From f14499d094443277f7acf4cfe1c20ef70d200d16 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sun, 17 Aug 2025 18:44:18 -0600 Subject: [PATCH 01/18] Interoperability with other Python binding frameworks --- CMakeLists.txt | 2 + include/pybind11/cast.h | 123 ++- include/pybind11/detail/class.h | 1 + include/pybind11/detail/common.h | 1 + include/pybind11/detail/foreign.h | 559 ++++++++++++++ include/pybind11/detail/internals.h | 107 ++- include/pybind11/detail/pymetabind.h | 709 ++++++++++++++++++ include/pybind11/detail/struct_smart_holder.h | 2 +- include/pybind11/detail/type_caster_base.h | 403 +++++++--- include/pybind11/embed.h | 9 +- include/pybind11/pybind11.h | 31 +- include/pybind11/pytypes.h | 14 +- include/pybind11/subinterpreter.h | 2 + tests/test_class_sh_property_non_owning.cpp | 2 + tests/test_multiple_interpreters.py | 15 +- 15 files changed, 1843 insertions(+), 137 deletions(-) create mode 100644 include/pybind11/detail/foreign.h create mode 100644 include/pybind11/detail/pymetabind.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ba5f665a2f..0e10b9198d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,11 +186,13 @@ set(PYBIND11_HEADERS include/pybind11/detail/descr.h include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h include/pybind11/detail/exception_translation.h + include/pybind11/detail/foreign.h include/pybind11/detail/function_record_pyobject.h include/pybind11/detail/init.h include/pybind11/detail/internals.h include/pybind11/detail/native_enum_data.h include/pybind11/detail/pybind11_namespace_macros.h + include/pybind11/detail/pymetabind.h include/pybind11/detail/struct_smart_holder.h include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index c635791fee..3b439d5496 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -862,6 +862,71 @@ struct holder_helper { static auto get(const T &p) -> decltype(p.get()) { return p.get(); } }; +struct holder_caster_foreign_helpers { + struct py_deleter { + void operator()(const void *) noexcept { + // Don't run the deleter if the interpreter has been shut down + if (!Py_IsInitialized()) + return; + gil_scoped_acquire guard; + Py_DECREF(o); + } + + PyObject *o; + }; + + template + static bool try_shared_from_this(std::enable_shared_from_this *value, + std::shared_ptr *holder_out) { + // object derives from enable_shared_from_this; + // try to reuse an existing shared_ptr if one is known + if (auto existing = try_get_shared_from_this(value)) { + *holder_out = existing; + return true; + } + return false; + } + + template + static bool try_shared_from_this(type *, std::shared_ptr *) { + return false; + } + + template + static bool set_foreign_holder(handle src, type *value, + std::shared_ptr *holder_out) { + // We only support using std::shared_ptr for foreign T, and + // it's done by creating a new shared_ptr control block that + // owns a reference to the original Python object. + if (value == nullptr) { + *holder_out = {}; + return true; + } + if (try_shared_from_this(value, holder_out)) { + return true; + } + *holder_out = std::shared_ptr(value, py_deleter{src.inc_ref().ptr()}); + return true; + } + + template + static bool set_foreign_holder(handle src, type *value, + std::shared_ptr *holder_out) { + std::shared_ptr holder_mut; + if (set_foreign_holder(src, value, &holder_mut)) { + *holder_out = holder_mut; + return true; + } + return false; + } + + template + static bool set_foreign_holder(handle, type *, ...) { + throw cast_error("Unable to cast foreign type to held instance -- " + "only std::shared_ptr is supported in this case"); + } +}; + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization. /// Type caster for holder types like std::shared_ptr, etc. /// The SFINAE hook is provided to help work around the current lack of support @@ -906,6 +971,11 @@ struct copyable_holder_caster : public type_caster_base { } } + bool set_foreign_holder(handle src) { + return holder_caster_foreign_helpers::set_foreign_holder( + src, (type *) value, &holder); + } + void load_value(value_and_holder &&v_h) { if (v_h.holder_constructed()) { value = v_h.value_ptr(); @@ -976,7 +1046,7 @@ struct copyable_holder_caster< } explicit operator std::shared_ptr *() { - if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + if (sh_load_helper.was_populated) { pybind11_fail("Passing `std::shared_ptr *` from Python to C++ is not supported " "(inherently unsafe)."); } @@ -984,14 +1054,14 @@ struct copyable_holder_caster< } explicit operator std::shared_ptr &() { - if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + if (sh_load_helper.was_populated) { shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, value); } return shared_ptr_storage; } std::weak_ptr potentially_slicing_weak_ptr() { - if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + if (sh_load_helper.was_populated) { // Reusing shared_ptr code to minimize code complexity. shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, @@ -1005,15 +1075,12 @@ struct copyable_holder_caster< static handle cast(const std::shared_ptr &src, return_value_policy policy, handle parent) { const auto *ptr = src.get(); - auto st = type_caster_base::src_and_type(ptr); - if (st.second == nullptr) { - return handle(); // no type info: error will be set already - } - if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + typename type_caster_base::cast_sources srcs{ptr}; + if (srcs.creates_smart_holder()) { return smart_holder_type_caster_support::smart_holder_from_shared_ptr( - src, policy, parent, st); + src, policy, parent, srcs.result); } - return type_caster_base::cast_holder(ptr, &src); + return type_caster_base::cast_holder(srcs, &src); } // This function will succeed even if the `responsible_parent` does not own the @@ -1040,6 +1107,11 @@ struct copyable_holder_caster< } } + bool set_foreign_holder(handle src) { + return holder_caster_foreign_helpers::set_foreign_holder( + src, (type *) value, &shared_ptr_storage); + } + void load_value(value_and_holder &&v_h) { if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = v_h; @@ -1077,6 +1149,7 @@ struct copyable_holder_caster< value = cast.second(sub_caster.value); if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + sh_load_helper.was_populated = true; } else { shared_ptr_storage = std::shared_ptr(sub_caster.shared_ptr_storage, (type *) value); @@ -1183,21 +1256,12 @@ struct move_only_holder_caster< static handle cast(std::unique_ptr &&src, return_value_policy policy, handle parent) { auto *ptr = src.get(); - auto st = type_caster_base::src_and_type(ptr); - if (st.second == nullptr) { - return handle(); // no type info: error will be set already - } - if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + typename type_caster_base::cast_sources srcs{ptr}; + if (srcs.creates_smart_holder()) { return smart_holder_type_caster_support::smart_holder_from_unique_ptr( - std::move(src), policy, parent, st); + std::move(src), policy, parent, srcs.result); } - return type_caster_generic::cast(st.first, - return_value_policy::take_ownership, - {}, - st.second, - nullptr, - nullptr, - std::addressof(src)); + return type_caster_base::cast_holder(srcs, &src); } static handle @@ -1223,6 +1287,12 @@ struct move_only_holder_caster< return false; } + bool set_foreign_holder(handle) { + throw cast_error("Foreign types cannot be converted to std::unique_ptr " + "because we don't know how to make them relinquish " + "ownership"); + } + void load_value(value_and_holder &&v_h) { if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = v_h; @@ -1281,6 +1351,7 @@ struct move_only_holder_caster< value = cast.second(sub_caster.value); if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + sh_load_helper.was_populated = true; } else { pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); @@ -2338,11 +2409,11 @@ object object_api::call(Args &&...args) const { PYBIND11_NAMESPACE_END(detail) template -handle type::handle_of() { +handle type::handle_of(bool foreign_ok) { static_assert(std::is_base_of>::value, - "py::type::of only supports the case where T is a registered C++ types."); + "py::type::of only supports the case where T is a registered C++ type."); - return detail::get_type_handle(typeid(T), true); + return detail::get_type_handle(typeid(T), true, foreign_ok); } #define PYBIND11_MAKE_OPAQUE(...) \ diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index cd7e87f845..e993e9e690 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -226,6 +226,7 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { } else { internals.registered_types_cpp.erase(tindex); } + get_foreign_internals().copy_move_ctors.erase(tindex); internals.registered_types_py.erase(tinfo->type); // Actually just `std::erase_if`, but that's only available in C++20 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index fac7c8c4dc..ed47a3f46d 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -348,6 +348,7 @@ #define PYBIND11_ENSURE_INTERNALS_READY \ { \ pybind11::detail::get_internals_pp_manager().unref(); \ + pybind11::detail::get_foreign_internals_pp_manager().unref(); \ pybind11::detail::get_internals(); \ } diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h new file mode 100644 index 0000000000..9a24ad0ba2 --- /dev/null +++ b/include/pybind11/detail/foreign.h @@ -0,0 +1,559 @@ +/* + pybind11/detail/foreign.h: Interoperability with other binding frameworks + + Copyright (c) 2025 Hudson River Trading + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "common.h" +#include "internals.h" +#include "type_caster_base.h" +#include "pymetabind.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// pybind11 exception translator that tries all known foreign ones +PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { + auto& foreign_internals = get_foreign_internals(); + for (pymb_framework *fw : foreign_internals.exc_frameworks) { + try { + fw->translate_exception(&p); + } catch (...) { + p = std::current_exception(); + } + } + std::rethrow_exception(p); +} + +// When learning about a new foreign type, should we automatically use it? +inline bool should_autoimport_foreign(foreign_internals &foreign_internals, + pymb_binding *binding) { + return foreign_internals.import_all && + binding->framework->abi_lang == pymb_abi_lang_cpp && + binding->framework->abi_extra == foreign_internals.self->abi_extra; +} + +// Add the given `binding` to our type maps so that we can use it to satisfy +// from- and to-Python requests for the given C++ type +inline void import_foreign_binding(pymb_binding *binding, + const std::type_info *cpptype) noexcept { + // Caller must hold the internals lock + auto &foreign_internals = get_foreign_internals(); + foreign_internals.imported_any = true; + foreign_internals.bindings.emplace(*cpptype, binding); +} + +// Callback functions for other frameworks to operate on our objects +// or tell us about theirs + +inline void *foreign_cb_from_python(pymb_binding *binding, + PyObject *pyobj, + uint8_t convert, + void (*keep_referenced)(void *ctx, + PyObject *obj), + void *keep_referenced_ctx) noexcept { +#if defined(PYBIND11_HAS_OPTIONAL) + using maybe_life_support = std::optional; +#else + struct maybe_life_support { + union { + loader_life_support supp; + }; + bool engaged = false; + + maybe_life_support() {} + maybe_life_support(maybe_life_support&) = delete; + loader_life_support* operator->() { return &supp; } + void emplace() { + new (&supp) loader_life_support(); + engaged = true; + } + ~maybe_life_support() { + if (engaged) { + supp.~loader_life_support(); + } + } + }; +#endif + maybe_life_support holder; + if (keep_referenced) { + holder.emplace(); + } + type_caster_generic caster{static_cast(binding->context)}; + void* ret = nullptr; + try { + if (caster.load(pyobj, convert)) { + ret = caster.value; + } + } catch (...) { + translate_exception(std::current_exception()); + PyErr_WriteUnraisable(pyobj); + } + if (keep_referenced) { + for (PyObject *item : holder->list_patients()) { + keep_referenced(keep_referenced_ctx, item); + } + } + return ret; +} + +inline PyObject *foreign_cb_to_python(pymb_binding *binding, + void *cobj, + enum pymb_rv_policy rvp_, + PyObject *parent) noexcept { + auto* ti = static_cast(binding->context); + if (cobj == nullptr) { + return none().release().ptr(); + } + auto rvp = static_cast(rvp_); + if (rvp > return_value_policy::reference_internal) { + // Treat out-of-range rvp as "return existing instance but don't + // make a new one", for compatibility with pymb_rv_policy_none + return find_registered_python_instance(cobj, ti).ptr(); + } + + copy_or_move_ctor copy_ctor = nullptr, move_ctor = nullptr; + if (rvp == return_value_policy::copy || rvp == return_value_policy::move) { + with_internals([&](internals&) { + auto& foreign_internals = get_foreign_internals(); + auto it = foreign_internals.copy_move_ctors.find(*ti->cpptype); + if (it != foreign_internals.copy_move_ctors.end()) { + std::tie(copy_ctor, move_ctor) = it->second; + } + }); + } + + try { + return type_caster_generic::cast(cobj, rvp, parent, ti, + copy_ctor, move_ctor).ptr(); + } catch (...) { + translate_exception(std::current_exception()); + return nullptr; + } +} + +inline int foreign_cb_keep_alive(PyObject *nurse, + void *payload, + void (*cb)(void*)) noexcept { + try { + if (!cb) { + keep_alive_impl(nurse, static_cast(payload)); + } else { + capsule patient{payload, cb}; + keep_alive_impl(nurse, patient); + } + return 0; + } catch (...) { + translate_exception(std::current_exception()); + return -1; + } +} + +inline void foreign_cb_translate_exception(const void *eptr) { + with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + // Try local translators. These don't have any special entries + // we need to skip. + std::exception_ptr e = *(const std::exception_ptr *) eptr; + for (auto &translator : local_exception_translators) { + try { + translator(e); + return; + } catch (...) { + e = std::current_exception(); + } + } + + // Try global translators, except the last one or two. + e = *(const std::exception_ptr *) eptr; + auto it = exception_translators.begin(); + auto leader = it; + // - The last one is the default translator. It translates + // standard exceptions, which we should leave up to the + // framework that bound a function. + ++leader; + // - If we've installed the foreign_exception_translator hook + // (for pybind11 functions to be able to translate other + // frameworks' exceptions), it's the second-last one and should + // be skipped too. We don't want mutual recursion between + // different frameworks' translators. + if (!get_foreign_internals().exc_frameworks.empty()) + ++leader; + + for (; leader != exception_translators.end(); ++it, ++leader) { + try { + (*it)(e); + return; + } catch (...) { + e = std::current_exception(); + } + } + + // Try the part of the default translator that is pybind11-specific + try { + std::rethrow_exception(e); + } catch (error_already_set &err) { + handle_nested_exception(err, e); + err.restore(); + return; + } catch (const builtin_exception &err) { + // Could not use template since it's an abstract class. + if (const auto *nep = dynamic_cast( + std::addressof(err))) { + handle_nested_exception(*nep, e); + } + err.set_error(); + return; + } + // Anything not caught by the above bubbles out. + }); +} + +inline void foreign_cb_add_foreign_binding(pymb_binding *binding) noexcept { + with_internals([&](internals&) { + auto& foreign_internals = get_foreign_internals(); + if (should_autoimport_foreign(foreign_internals, binding)) { + import_foreign_binding( + binding, (const std::type_info *) binding->native_type); + } + }); +} + +inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { + with_internals([&](internals&) { + auto& foreign_internals = get_foreign_internals(); + auto remove_from_type = [&](const std::type_info *type) { + auto range = foreign_internals.bindings.equal_range(*type); + for (auto it = range.first; it != range.second; ++it) { + if (it->second == binding) { + foreign_internals.bindings.erase(it); + break; + } + } + }; + bool should_remove_auto = + should_autoimport_foreign(foreign_internals, binding); + auto it = foreign_internals.manual_imports.find(binding); + if (it != foreign_internals.manual_imports.end()) { + remove_from_type(it->second); + should_remove_auto &= (it->second != binding->native_type); + foreign_internals.manual_imports.erase(it); + } + if (should_remove_auto) + remove_from_type((const std::type_info *) binding->native_type); + }); +} + +inline void foreign_cb_add_foreign_framework(pymb_framework *framework) + noexcept { + if (framework->translate_exception) { + with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &) { + auto& foreign_internals = get_foreign_internals(); + if (foreign_internals.exc_frameworks.empty()) { + // First foreign framework with an exception translator. + // Add our `foreign_exception_translator` wrapper in the + // 2nd-last position (last is the default exception + // translator). + auto leader = exception_translators.begin(); + auto trailer = exception_translators.before_begin(); + while (++leader != exception_translators.end()) + ++trailer; + exception_translators.insert_after( + trailer, foreign_exception_translator); + } + // Add the new framework at the end of the list + auto leader = foreign_internals.exc_frameworks.begin(); + auto trailer = leader; + while (++leader != foreign_internals.exc_frameworks.end()) + ++trailer; + foreign_internals.exc_frameworks.insert_after( + trailer, framework); + }); + } +} + +// (end of callbacks) + +// Advertise our existence, and the above callbacks, to other frameworks +PYBIND11_NOINLINE bool foreign_internals::initialize() { + bool inited_by_us = with_internals([&](internals&) { + if (registry) + return false; + registry = pymb_get_registry(); + if (!registry) + throw error_already_set(); + + self = std::make_unique(); + self->name = "pybind11 " PYBIND11_ABI_TAG; + // TODO: pybind11 does leak some bindings; there should be a way to + // indicate that (so that eg nanobind can disable its leak detection) + // without promising to leak all bindings + self->bindings_usable_forever = 0; + self->abi_lang = pymb_abi_lang_cpp; + self->abi_extra = PYBIND11_PLATFORM_ABI_ID; + self->from_python = foreign_cb_from_python; + self->to_python = foreign_cb_to_python; + self->keep_alive = foreign_cb_keep_alive; + self->translate_exception = foreign_cb_translate_exception; + self->add_foreign_binding = foreign_cb_add_foreign_binding; + self->remove_foreign_binding = foreign_cb_remove_foreign_binding; + self->add_foreign_framework = foreign_cb_add_foreign_framework; + return true; + }); + if (inited_by_us) { + // Unlock internals before calling add_framework, so that the callbacks + // (foreign_cb_add_foreign_binding, etc) can safely re-lock it. + pymb_add_framework(registry, self.get()); + } + return inited_by_us; +} + +inline foreign_internals::~foreign_internals() = default; + +// Learn to satisfy from- and to-Python requests for `cpptype` using the +// foreign binding provided by the given `pytype`. If cpptype is nullptr, infer +// the C++ type by looking at the binding, and require that its ABI match ours. +// Throws an exception on failure. Caller must hold the internals lock and have +// already called foreign_internals.initialize_if_needed(). +PYBIND11_NOINLINE void import_foreign_type(type pytype, + const std::type_info *cpptype) { + auto &foreign_internals = get_foreign_internals(); + pymb_binding* binding = pymb_get_binding(pytype.ptr()); + if (!binding) + pybind11_fail("pybind11::import_foreign_type(): type does not define " + "a __pymetabind_binding__"); + if (binding->framework == foreign_internals.self.get()) + pybind11_fail("pybind11::import_foreign_type(): type is not foreign"); + if (!cpptype) { + if (binding->framework->abi_lang != pymb_abi_lang_cpp) + pybind11_fail("pybind11::import_foreign_type(): type is not " + "written in C++, so you must specify a C++ type"); + if (binding->framework->abi_extra != foreign_internals.self->abi_extra) + pybind11_fail("pybind11::import_foreign_type(): type has " + "incompatible C++ ABI with this module"); + cpptype = (const std::type_info *) binding->native_type; + } + + auto result = foreign_internals.manual_imports.emplace(binding, cpptype); + if (!result.second) { + auto *existing = (const std::type_info *) result.first->second; + if (existing != cpptype && *existing != *cpptype) + pybind11_fail("pybind11::import_foreign_type(): type was " + "already imported as a different C++ type"); + } + import_foreign_binding(binding, cpptype); +} + +// Call `import_foreign_binding()` for every ABI-compatible type provided by +// other C++ binding frameworks used by extension modules loaded in this +// interpreter, both those that exist now and those bound in the future. +PYBIND11_NOINLINE void foreign_enable_import_all() { + auto& foreign_internals = get_foreign_internals(); + bool proceed = with_internals([&](internals&) { + if (foreign_internals.import_all) + return false; + foreign_internals.import_all = true; + return true; + }); + if (!proceed) + return; + if (foreign_internals.initialize_if_needed()) { + // pymb_add_framework tells us about every existing type when we + // register, so if we register with import enabled, we're done + return; + } + // If we enable import after registering, we have to iterate over the + // list of types ourselves. Do this without the internals lock held so + // we can reuse the pymb callback functions. foreign_internals registry + + // self never change once they're non-null, so we can accesss them + // without locking here. + pymb_lock_registry(foreign_internals.registry); + PYMB_LIST_FOREACH(struct pymb_binding*, binding, + foreign_internals.registry->bindings) { + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding)) { + foreign_cb_add_foreign_binding(binding); + pymb_unref_binding(binding); + } + } + pymb_unlock_registry(foreign_internals.registry); +} + +// Expose hooks for other frameworks to use to work with the given pybind11 +// type object. Caller must hold the internals lock and have already called +// foreign_internals.initialize_if_needed(). +PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { + auto& foreign_internals = get_foreign_internals(); + auto range = foreign_internals.bindings.equal_range(*ti->cpptype); + for (auto it = range.first; it != range.second; ++it) + if (it->second->framework == foreign_internals.self.get()) + return; // already exported + + auto *binding = new pymb_binding{}; + binding->framework = foreign_internals.self.get(); + binding->pytype = ti->type; + binding->native_type = ti->cpptype; + binding->source_name = strdup(clean_type_id(ti->cpptype->name()).c_str()); + binding->context = ti; + + capsule tie_lifetimes((void *) binding, [](void *p) { + pymb_binding *binding = (pymb_binding *) p; + pymb_remove_binding(get_foreign_internals().registry, binding); + free(const_cast(binding->source_name)); + delete binding; + }); + keep_alive_impl((PyObject *) ti->type, tie_lifetimes); + + foreign_internals.bindings.emplace(*ti->cpptype, binding); + pymb_add_binding(foreign_internals.registry, binding); +} + +// Call `export_type_to_foreign()` for each type that currently exists in our +// internals structure and each type created in the future. +PYBIND11_NOINLINE void foreign_enable_export_all() { + auto& foreign_internals = get_foreign_internals(); + bool proceed = with_internals([&](internals&) { + if (foreign_internals.export_all) + return false; + foreign_internals.export_all = true; + foreign_internals.export_type_to_foreign = + &detail::export_type_to_foreign; + return true; + }); + if (!proceed) + return; + foreign_internals.initialize_if_needed(); + with_internals([&](internals& internals) { + for (const auto& entry : internals.registered_types_cpp) { + detail::export_type_to_foreign(entry.second); + } + }); +} + +// Invoke `attempt(closure, binding)` for each foreign binding `binding` +// that claims `type` and was not supplied by us, until one of them returns +// non-null. Return that first non-null value, or null if all attempts failed. +PYBIND11_NOINLINE void* try_foreign_bindings( + const std::type_info *type, + void* (*attempt)(void *closure, pymb_binding *binding), + void *closure) { + auto &internals = get_internals(); + auto &foreign_internals = get_foreign_internals(); + + PYBIND11_LOCK_INTERNALS(internals); + auto range = foreign_internals.bindings.equal_range(*type); + + if (range.first == range.second) + return nullptr; // no foreign bindings + + if (std::next(range.first) == range.second) { + // Single binding - check that it's not our own + auto *binding = range.first->second; + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding)) { +#ifdef Py_GIL_DISABLED + // attempt() might execute Python code; drop the internals lock + // to avoid a deadlock + lock.unlock(); +#endif + void *result = attempt(closure, binding); + pymb_unref_binding(binding); + return result; + } + return nullptr; + } + + // Multiple bindings - try all except our own +#ifndef Py_GIL_DISABLED + for (auto it = range.first; it != range.second; ++it) { + auto *binding = it->second; + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding)) { + void *result = attempt(closure, binding); + pymb_unref_binding(binding); + if (result) + return result; + } + } + return nullptr; +#else + // In free-threaded mode, this is tricky: we need to drop the + // internals lock before calling attempt(), but once we do so, + // any of these bindings that might be in the middle of getting deleted + // can be concurrently removed from the map, which would interfere + // with our iteration. Copy the binding pointers out of the list to avoid + // this problem. + + // Count the number of foreign bindings we might see + size_t len = (size_t) std::distance(range.first, range.second); + + // Allocate temporary storage for that many pointers + pymb_binding **scratch = + (pymb_binding **) alloca(len * sizeof(pymb_binding*)); + pymb_binding **scratch_tail = scratch; + + // Iterate again, taking out strong references and saving pointers to + // our scratch storage + for (auto it = range.first; it != range.second; ++it) { + auto *binding = it->second; + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding)) + *scratch_tail++ = binding; + } + + // Drop the lock and proceed using only our saved binding pointers. + // Since we obtained strong references to them, there is no remaining + // concurrent-destruction hazard. + lock.unlock(); + void *result = nullptr; + while (scratch != scratch_tail) { + if (!result) + result = attempt(closure, *scratch); + pymb_unref_binding(*scratch); + ++scratch; + } + return result; +#endif +} + +PYBIND11_NAMESPACE_END(detail) + +inline void set_foreign_type_defaults(bool export_all, bool import_all) { + auto &foreign_internals = detail::get_foreign_internals(); + if (import_all && !foreign_internals.import_all) + detail::foreign_enable_import_all(); + if (export_all && !foreign_internals.export_all) + detail::foreign_enable_export_all(); +} + +template +inline void import_foreign_type(type pytype) { + const std::type_info *cpptype = std::is_void::value ? nullptr : &typeid(T); + auto& foreign_internals = detail::get_foreign_internals(); + foreign_internals.initialize_if_needed(); + detail::with_internals([&](detail::internals&) { + detail::import_foreign_type(pytype, cpptype); + }); +} + +inline void export_type_to_foreign(type ty) { + detail::type_info *ti = detail::get_type_info((PyTypeObject *) ty.ptr()); + if (!ti) + pybind11_fail("pybind11::export_type_to_foreign: not a " + "pybind11 registered type"); + auto& foreign_internals = detail::get_foreign_internals(); + foreign_internals.initialize_if_needed(); + detail::with_internals([&](detail::internals&) { + detail::export_type_to_foreign(ti); + }); +} + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 46f44d9b10..77746f18c7 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -22,6 +22,10 @@ #include #include +struct pymb_binding; +struct pymb_framework; +struct pymb_registry; + /// Tracks the `internals` and `type_info` ABI version independent of the main library version. /// /// Some portions of the code use an ABI that is conditional depending on this @@ -176,6 +180,8 @@ struct type_equal_to { template using type_map = std::unordered_map; +template +using type_multimap = std::unordered_multimap; struct override_hash { inline size_t operator()(const std::pair &v) const { @@ -290,6 +296,82 @@ struct internals { ~internals() = default; }; +using copy_or_move_ctor = void *(*) (const void *); + +// Information to support working with types from other binding frameworks or +// other ABI versions of pybind11, and sharing our types with them. This should +// logically be part of `internals`, but we want to support it without an ABI +// version bump. If it gets incorpoprated into `internals` in a later ABI +// version, it would be good for performance to also add a flag to `type_info` +// indicating whether any foreign bindings are also known for its C++ type; +// that way we can avoid an extra lookup when conversion to a native type fails. +struct foreign_internals { + // Registered foreign bindings for each C++ type. + // Protected by internals::mutex. + type_multimap bindings; + + // For each C++ type with a native binding, store pointers to its + // copy and move constructors. These would ideally move inside `type_info` + // on an ABI bump. Protected by internals::mutex. + type_map> copy_move_ctors; + + // List of frameworks that provide exception translators. + // Protected by internals::exception_translator_mutex. + // If this is non-empty, there is a single translator in the penultimate + // position of internals::registered_exception_translators that calls the + // translate_exception methods of each framework in this list. + std::forward_list exc_frameworks; + + // Pointer to the registry of foreign bindings. + // Protected by internals::mutex; constant once becoming non-null. + pymb_registry *registry = nullptr; + + // Hooks allowing other frameworks to interact with us. + // Protected by internals::mutex; constant once becoming non-null. + std::unique_ptr self; + + // Remember the C++ type associated with each binding by + // import_type_from_foreign(), so we can clean up `bindings` properly. + // Protected by internals::mutex. + std::unordered_map manual_imports; + + // Pointer to `detail::export_type_to_foreign` in foreign.h, or nullptr if + // export_all is false. This indirection is vital to avoid having every + // compilation unit with a py::class_ pull in the callback methods in + // foreign.h. Instead, only compilation units that call + // set_foreign_type_defaults(), import_foreign_type(), or + // export_type_to_foreign() will emit that code. + void (*export_type_to_foreign)(type_info *); + + // Should we automatically advertise our types to other binding frameworks, + // or only when requested via pybind11::export_type_to_foreign()? + // Never becomes false once it is set to true. + bool export_all = false; + + // Should we automatically use types advertised by other frameworks as + // a fallback when we can't do a cast using pybind11 types, or only when + // requested via pybind11::import_foreign_type()? + // Never becomes false once it is set to true. + bool import_all = false; + + // Are there any entries in `bindings` that don't correspond to our + // own types? + bool imported_any = false; + + inline ~foreign_internals(); + + // Returns true if we initialized, false if someone else already did. + inline bool initialize_if_needed() { + if (registry) + return false; + return initialize(); + } + + private: + inline bool initialize(); +}; + // the internals struct (above) is shared between all the modules. local_internals are only // for a single module. Any changes made to internals may require an update to // PYBIND11_INTERNALS_VERSION, breaking backwards compatibility. local_internals is, by design, @@ -343,13 +425,11 @@ struct type_info { bool module_local : 1; }; -#define PYBIND11_INTERNALS_ID \ - "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" +#define PYBIND11_ABI_TAG "v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID -#define PYBIND11_MODULE_LOCAL_ID \ - "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" +#define PYBIND11_INTERNALS_ID "__pybind11_internals_" PYBIND11_ABI_TAG "__" +#define PYBIND11_MODULE_LOCAL_ID "__pybind11_module_local_v" PYBIND11_ABI_TAG "__" inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) @@ -691,6 +771,21 @@ inline auto with_exception_translators(const F &cb) local_internals.registered_exception_translators); } +inline internals_pp_manager &get_foreign_internals_pp_manager() { + static internals_pp_manager foreign_internals_pp_manager( + PYBIND11_INTERNALS_ID "foreign", nullptr); + return foreign_internals_pp_manager; +} + +inline foreign_internals &get_foreign_internals() { + auto &ppmgr = get_foreign_internals_pp_manager(); + auto &ptr = *ppmgr.get_pp(); + if (!ptr) { + ptr.reset(new foreign_internals()); + } + return *ptr; +} + inline std::uint64_t mix64(std::uint64_t z) { // David Stafford's variant 13 of the MurmurHash3 finalizer popularized // by the SplitMix PRNG. diff --git a/include/pybind11/detail/pymetabind.h b/include/pybind11/detail/pymetabind.h new file mode 100644 index 0000000000..d42cd9c7eb --- /dev/null +++ b/include/pybind11/detail/pymetabind.h @@ -0,0 +1,709 @@ +/* + * pymetabind.h: definitions for interoperability between different + * Python binding frameworks + * + * Copy this header file into the implementation of a framework that uses it. + * This functionality is intended to be used by the framework itself, + * rather than by users of the framework. + * + * This is version 0.1 of pymetabind. Changelog: + * + * Version 0.1: Initial draft. ABI may change without warning while we + * 2025-08-16 prove out the concept. Please wait for a 1.0 release + * before including this header in a published release of + * any binding framework. + * + * Copyright (c) 2025 Hudson River Trading + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include +#include + +#if !defined(PY_VERSION_HEX) +# error You must include Python.h before this header +#endif + +/* + * There are two ways to use this header file. The default is header-only style, + * where all functions are defined as `inline`. If you want to emit functions + * as non-inline, perhaps so you can link against them from non-C/C++ code, + * then do the following: + * - In every compilation unit that includes this header, `#define PYMB_FUNC` + * first. (The `PYMB_FUNC` macro will be expanded in place of the "inline" + * keyword, so you can also use it to add any other declaration attributes + * required by your environment.) + * - In all those compilation units except one, also `#define PYMB_DECLS_ONLY` + * before including this header. The definitions will be emitted in the + * compilation unit that doesn't request `PYMB_DECLS_ONLY`. + */ +#if !defined(PYMB_FUNC) +#define PYMB_FUNC inline +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +/* + * Approach used to cast a previously unknown C++ instance into a Python object. + * The values of these enumerators match those for `nanobind::rv_policy` and + * `pybind11::return_value_policy`. + */ +enum pymb_rv_policy { + // (Values 0 and 1 correspond to `automatic` and `automatic_reference`, + // which should become one of the other policies before reaching us) + + // Create a Python object that owns a pointer to heap-allocated storage + // and will destroy and deallocate it when the Python object is destroyed + pymb_rv_policy_take_ownership = 2, + + // Create a Python object that owns a new C++ instance created via + // copy construction from the given one + pymb_rv_policy_copy = 3, + + // Create a Python object that owns a new C++ instance created via + // move construction from the given one + pymb_rv_policy_move = 4, + + // Create a Python object that wraps the given pointer to a C++ instance + // but will not destroy or deallocate it + pymb_rv_policy_reference = 5, + + // `reference`, plus arrange for the given `parent` python object to + // live at least as long as the new object that wraps the pointer + pymb_rv_policy_reference_internal = 6, + + // Don't create a new Python object; only try to look up an existing one + // from the same framework + pymb_rv_policy_none = 7 +}; + +/* + * The language to which a particular framework provides bindings. Each + * language has its own semantics for how to interpret + * `pymb_framework::abi_extra` and `pymb_binding::native_type`. + */ +enum pymb_abi_lang { + // C. `pymb_framework::abi_extra` and `pymb_binding::native_type` are NULL. + pymb_abi_lang_c = 1, + + // C++. `pymb_framework::abi_extra` is in the format used by + // nanobind since 2.6.1 (NB_PLATFORM_ABI_TAG in nanobind/src/nb_abi.h) + // and pybind11 since 2.11.2/2.12.1/2.13.6 (PYBIND11_PLATFORM_ABI_ID in + // pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h). + // `pymb_binding::native_type` is a cast `const std::type_info*` pointer. + pymb_abi_lang_cpp = 2, + + // extensions welcome! +}; + +/* + * Simple linked list implementation. `pymb_list_node` should be the first + * member of a structure so you can downcast it to the appropriate type. + */ +struct pymb_list_node { + struct pymb_list_node *next; + struct pymb_list_node *prev; +}; + +struct pymb_list { + struct pymb_list_node head; +}; + +inline void pymb_list_init(struct pymb_list* list) { + list->head.prev = list->head.next = &list->head; +} + +inline void pymb_list_unlink(struct pymb_list_node* node) { + if (node->next) { + node->next->prev = node->prev; + node->prev->next = node->next; + node->next = node->prev = NULL; + } +} + +inline void pymb_list_append(struct pymb_list* list, + struct pymb_list_node* node) { + pymb_list_unlink(node); + struct pymb_list_node* tail = list->head.prev; + tail->next = node; + list->head.prev = node; + node->prev = tail; + node->next = &list->head; +} + +#define PYMB_LIST_FOREACH(type, name, list) \ + for (type name = (type) (list).head.next; \ + name != (type) &(list).head; \ + name = (type) name->hook.next) + +/* + * The registry holds information about all the interoperable binding + * frameworks and individual type bindings that are loaded in a Python + * interpreter process. It is protected by a mutex in free-threaded builds, + * and by the GIL in regular builds. + * + * The only data structure we use is a C doubly-linked list, which offers a + * lowest-common-denominator ABI and cheap addition and removal. It is expected + * that individual binding frameworks will use their `add_foreign_binding` and + * `remove_foreign_binding` callbacks to maintain references to these structures + * in more-performant private data structures of their choosing. + * + * The pointer to the registry is stored in a Python capsule object with type + * "pymetabind_registry", which is stored in the PyInterpreterState_GetDict() + * under the string key "__pymetabind_registry__". Any ABI-incompatible changes + * after v1.0 (which we hope to avoid!) will result in a new name for the + * dictionary key. You can obtain a registry pointer using + * `pymb_get_registry()`, defined below. + */ +struct pymb_registry { + // Linked list of registered `pymb_framework` structures + struct pymb_list frameworks; + + // Linked list of registered `pymb_binding` structures + struct pymb_list bindings; + + // Reserved for future extensions; currently set to 0 + uint32_t reserved; + +#if defined(Py_GIL_DISABLED) + // Mutex guarding accesses to `frameworks` and `bindings`. + // On non-free-threading builds, these are guarded by the Python GIL. + PyMutex mutex; +#endif +}; + +#if defined(Py_GIL_DISALED) +inline void pymb_lock_registry(struct pymb_registry* registry) { + PyMutex_Lock(®istry->mutex); +} +inline void pymb_unlock_registry(struct pymb_registry* registry) { + PyMutex_Unlock(®istry->mutex); +} +#else +inline void pymb_lock_registry(struct pymb_registry*) {} +inline void pymb_unlock_registry(struct pymb_registry*) {} +#endif + +struct pymb_binding; + +/* + * Information about one framework that has registered itself with pymetabind. + * "Framework" here refers to a set of bindings that are natively mutually + * interoperable. So, different binding libraries would be different frameworks, + * as would versions of the same library that use incompatible data structures + * due to ABI changes or build flags. + * + * A framework that wishes to either export bindings (allow other frameworks + * to perform to/from Python conversion for its types) or import bindings + * (perform its own to/from Python conversion for other frameworks' types) + * must start by creating and filling out a `pymb_framework` structure. + * This can be allocated in any way that the framework prefers (e.g., on + * the heap or in static storage). Once filled out, the framework structure + * should be passed to `pymb_add_framework()`. It must then remain accessible + * and unmodified (except as documented below) until the Python interpreter + * is finalized. After finalization, such as in a `Py_AtExit` handler, if + * all bindings have been removed already, you may optionally clean up by + * calling `pymb_list_unlink(&framework->hook)` and then deallocating the + * `pymb_framework` structure. + * + * All fields of this structure are set before it is made visible to other + * threads and then never changed, so they don't need locking to access. + * Some methods require locking or other synchronization to call; see their + * individual documentation. + */ +struct pymb_framework { + // Hook by which this structure is linked into the list of + // `pymb_registry::frameworks`. May be modified as other frameworks are + // added; protected by the `pymb_registry::mutex` in free-threaded builds. + struct pymb_list_node hook; + + // Human-readable description of this framework, as a NUL-terminated string + const char* name; + + // Does this framework guarantee that its `pymb_binding` structures remain + // valid to use for the lifetime of the Python interpreter process once + // they have been linked into the lists in `pymb_registry`? Setting this + // to true reduces the number of atomic operations needed to work with + // this framework's bindings in free-threaded builds. + uint8_t bindings_usable_forever; + + // Reserved for future extensions. Set to 0. + uint8_t reserved[3]; + + // The language to which this framework provides bindings: one of the + // `pymb_abi_lang` enumerators. + enum pymb_abi_lang abi_lang; + + // NUL-terminated string constant encoding additional information that must + // match in order for two types with the same `abi_lang` to be usable from + // each other's environments. See documentation of `abi_lang` enumerators + // for language-specific guidance. This may be NULL if there are no + // additional ABI details that are relevant for your language. + // + // This is only the platform details that affect things like the layout + // of objects provided by the `abi_lang` (std::string, etc); Python build + // details (free-threaded, stable ABI, etc) should not impact this string. + // Details that are already guaranteed to match by virtue of being in the + // same address space -- architecture, pointer size, OS -- also should not + // impact this string. + // + // For efficiency, `pymb_add_framework()` will compare this against every + // other registered framework's `abi_extra` tag, and re-point an incoming + // framework's `abi_extra` field to refer to the matching `abi_extra` string + // of an already-registered framework if one exists. This acts as a simple + // form of interning to speed up checking that a given binding is usable. + // Thus, to check whether another framework's ABI matches yours, you can + // do a pointer comparison `me->abi_extra == them->abi_extra`. + const char* abi_extra; + + // The function pointers below allow other frameworks to interact with + // bindings provided by this framework. They are constant after construction + // and, except for `translate_exception()`, must not throw C++ exceptions. + // Unless otherwise documented, they must not be NULL. + + // Extract a C/C++/etc object from `pyobj`. The desired type is specified by + // providing a `pymb_binding*` for some binding that belongs to this + // framework. Return a pointer to the object, or NULL if no pointer of the + // appropriate type could be extracted. + // + // If `convert` is nonzero, be more willing to perform implicit conversions + // to make the cast succeed; the intent is that one could perform overload + // resolution by doing a first pass with convert=false to find an exact + // match, and then a second with convert=true to find an approximate match + // if there's no exact match. + // + // If `keep_referenced` is not NULL, then `from_python` may make calls to + // `keep_referenced` to request that some Python objects remain referenced + // until the returned object is no longer needed. The `keep_referenced_ctx` + // will be passed as the first argument to any such calls. + // `keep_referenced` should incref its `obj` immediately and remember + // that it should be decref'ed later, for no net change in refcount. + // This is an abstraction around something like the cleanup_list in + // nanobind or loader_life_support in pybind11. + // + // On free-threaded builds, callers must ensure that the `binding` is not + // destroyed during a call to `from_python`. The requirements for this are + // subtle; see the full discussion in the comment for `struct pymb_binding`. + void* (*from_python)(struct pymb_binding* binding, + PyObject* pyobj, + uint8_t convert, + void (*keep_referenced)(void* ctx, PyObject* obj), + void* keep_referenced_ctx); + + // Wrap the C/C++/etc object `cobj` into a Python object using the given + // return value policy. The type is specified by providing a `pymb_binding*` + // for some binding that belongs to this framework. `parent` is relevant + // only if `rvp == pymb_rv_policy_reference_internal`. rvp must be one of + // the defined enumerators. Returns NULL if the cast is not possible, or + // a new reference otherwise. + // + // A NULL return may leave the Python error indicator set if something + // specifically describable went wrong during conversion, but is not + // required to; returning NULL without PyErr_Occurred() should be + // interpreted as a generic failure to convert `cobj` to a Python object. + // + // On free-threaded builds, callers must ensure that the `binding` is not + // destroyed during a call to `to_python`. The requirements for this are + // subtle; see the full discussion in the comment for `struct pymb_binding`. + PyObject* (*to_python)(struct pymb_binding* binding, + void* cobj, + enum pymb_rv_policy rvp, + PyObject* parent); + + // Request that a PyObject reference be dropped, or that a callback + // be invoked, when `nurse` is destroyed. `nurse` should be an object + // whose type is bound by this framework. If `cb` is NULL, then + // `payload` is a PyObject* to decref; otherwise `payload` will + // be passed as the argument to `cb`. Returns 0 if successful, + // or -1 and sets the Python error indicator on error. + // + // No synchronization is required to call this method. + int (*keep_alive)(PyObject* nurse, void* payload, void (*cb)(void*)); + + // Attempt to translate a C++ exception known to this framework to Python. + // This should translate only framework-specific exceptions or user-defined + // exceptions that were registered with the framework, not generic + // ones such as `std::exception`. If successful, return normally with the + // Python error indicator set; otherwise, reraise the provided exception. + // `eptr` should be cast to `const std::exception_ptr* eptr` before use. + // This function pointer may be NULL if this framework does not provide + // C++ exception translation. + // + // No synchronization is required to call this method. + void (*translate_exception)(const void* eptr); + + // Notify this framework that some other framework published a new binding. + // This call will be made after the new binding has been linked into the + // `pymb_registry::bindings` list. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*add_foreign_binding)(struct pymb_binding* binding); + + // Notify this framework that some other framework is about to remove + // a binding. This call will be made after the binding has been removed + // from the `pymb_registry::bindings` list. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*remove_foreign_binding)(struct pymb_binding* binding); + + // Notify this framework that some other framework came into existence. + // This call will be made after the new framework has been linked into the + // `pymb_registry::frameworks` list and before it adds any bindings. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*add_foreign_framework)(struct pymb_framework* framework); + + // There is no remove_foreign_framework(); the interpreter has + // already been finalized at that point, so there's nothing for the + // callback to do. +}; + +/* + * Information about one type binding that belongs to a registered framework. + * + * A framework that binds some type and wants to allow other frameworks to + * work with objects of that type must create a `pymb_binding` structure for + * the type. This can be allocated in any way that the framework prefers (e.g., + * on the heap or within the type object). Once filled out, the binding + * structure should be passed to `pymb_add_binding()`. If the Python type object + * underlying the binding is to be deallocated, a `pymb_remove_binding()` call + * must be made, and the `pymb_binding` structure cannot be deallocated until + * `pymb_remove_binding()` returns. The call to `pymb_remove_binding()` + * must occur *during* deallocation of the binding's Python type object, i.e., + * at a time when `Py_REFCNT(pytype) == 0` but the storage for `pytype` is not + * yet eligible to be reused for another object. Many frameworks use a custom + * metaclass, and can add the call to `pymb_remove_binding()` from the metaclass + * `tp_dealloc`; those that don't can use a weakref callback on the type object + * instead. The constraint on destruction timing allows `pymb_try_ref_binding()` + * to temporarily prevent the binding's destruction by incrementing the type + * object's reference count. + * + * Each Python type object for which a `pymb_binding` exists will have an + * attribute "__pymetabind_binding__" whose value is a capsule object + * that contains the `pymb_binding` pointer under the name "pymetabind_binding". + * The attribute is set during `pymb_add_binding()`. This is provided to allow: + * - Determining which framework to call for a foreign `keep_alive` operation + * - Locating `pymb_binding` objects for types written in a different language + * than yours (where you can't look up by the `pymb_binding::native_type`), + * so that you can work with their contents using non-Python-specific + * cross-language support + * - Extracting the native object from a Python object without being too picky + * about what type it is (risky, but maybe you have out-of-band information + * that shows it's safe) + * The preferred mechanism for same-language object access is to maintain a + * hashtable keyed on `pymb_binding::native_type` and look up the binding for + * the type you want/have. Compared to reading the capsule, this better + * supports inheritance, to-Python conversions, and implicit conversions, and + * it's probably also faster depending on how it's implemented. + * + * It is valid for multiple frameworks to claim (in separate bindings) the + * same C/C++ type, or even the same Python type. (A case where multiple + * frameworks would bind the same Python type is if one is acting as an + * extension to the other, such as to support extracting pointers to + * non-primary base classes when the base framework doesn't think about + * such things.) If multiple frameworks claim the same Python type, then each + * new registrant will replace the "__pymetabind_binding__" capsule and there + * is no way to locate the other bindings from the type object. + * + * All fields of this structure are set before it is made visible to other + * threads and then never changed, so they don't need locking to access. + * However, on free-threaded builds it is necessary to validate that the type + * object is not partway through being destroyed before you use the binding, + * and prevent such destruction from beginning until you're done. To do so, + * call `pymb_try_ref_binding()`; if it returns false, don't use the binding, + * else use it and then call `pymb_unref_binding()` when done. + * (On non-free-threaded builds, these do incref/decref to prevent destruction + * of the type from starting, but can't fail because there's no *concurrent* + * destruction hazard.) + * + * In order to work with one framework's Python objects of a certain type, other + * frameworks must be able to locate a `pymb_binding` structure for that type. + * It is expected that they will maintain their own type-to-binding maps, which + * they can keep up-to-date via their `pymb_framework::add_foreign_binding` and + * `pymb_framework::remove_foreign_binding` hooks. It is important to think very + * carefully about how to design the synchronization for these maps so that + * lookups do not return pointers to bindings that have been deallocated. + * The remainder of this comment provides some suggestions. + * + * The recommended way to handle synchronization is to protect your type lookup + * map with a readers/writer lock. In your `remove_foreign_binding` hook, + * obtain a write lock, and hold it while removing the corresponding entry from + * the map. Before performing a type lookup, obtain a read lock. If the lookup + * succeeds, call `pymb_try_ref_binding()` on the resulting binding before + * you release your read lock. Since the binding structure can't be deallocated + * until all `remove_foreign_binding` hooks have returned, this scheme provides + * effective protection. It is important not to hold the read lock while + * executing arbitrary Python code, since a deadlock would result if the type + * object is deallocated (requiring a write lock) while the read lock were held. + * Note that `pymb_framework::from_python` for many popular frameworks is + * capable of executing arbitrary Python code to perform implicit conversions. + * + * The lock on a single shared type lookup map is a contention bottleneck, + * especially if you don't have a readers/writer lock and wish to get by with + * an ordinary mutex. To improve performance, you can give each thread its + * own lookup map, and require `remove_foreign_binding` to update all of them. + * As long as the per-thread maps are always visited in a consistent order + * when removing a binding, the splitting shouldn't introduce new deadlocks. + * Since each thread has a separate mutex for its separate map, contention + * occurs only when bindings are being added or removed, which is much less + * common than using them. + */ +struct pymb_binding { + // Hook by which this structure is linked into the list of + // `pymb_registry::bindings` + struct pymb_list_node hook; + + // The framework that provides this binding + struct pymb_framework* framework; + + // Python type: you will get an instance of this type from a successful + // call to `framework::from_python()` that passes this binding + PyTypeObject* pytype; + + // The native identifier for this type in `framework->abi_lang`, if that is + // a concept that exists in that language. See the documentation of + // `enum pymb_abi_lang` for specific per-language semantics. + const void* native_type; + + // The way that this type would be written in `framework->abi_lang` source + // code, as a NUL-terminated byte string without struct/class/enum words. + // Examples: "Foo", "Bar::Baz", "std::vector >" + const char* source_name; + + // Pointer that is free for use by the framework, e.g., to point to its + // own data about this type. If the framework needs more data, it can + // over-allocate the `pymb_binding` storage and use the space after this. + void* context; +}; + +/* + * Users of non-C/C++ languages are welcome to replicate the logic of these + * inline functions rather than calling them. Their implementations are + * considered part of the ABI. + */ + +PYMB_FUNC struct pymb_registry* pymb_get_registry(); +PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, + struct pymb_framework* framework); +PYMB_FUNC void pymb_remove_framework(struct pymb_registry* registry, + struct pymb_framework* framework); +PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, + struct pymb_binding* binding); +PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, + struct pymb_binding* binding); +PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding); +PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding); +PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); + +#if !defined(PYMB_DECLS_ONLY) + +/* + * Locate an existing `pymb_registry`, or create a new one if necessary. + * Returns a pointer to it, or NULL with the CPython error indicator set. + * This must be called from a module initialization function so that the + * import lock can provide mutual exclusion. + */ +PYMB_FUNC struct pymb_registry* pymb_get_registry() { +#if defined(PYPY_VERSION) + PyObject* dict = PyEval_GetBuiltins(); +#elif PY_VERSION_HEX < 0x03090000 + PyObject* dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); +#else + PyObject* dict = PyInterpreterState_GetDict(PyInterpreterState_Get()); +#endif + PyObject* key = PyUnicode_FromString("__pymetabind_registry__"); + if (!dict || !key) { + Py_XDECREF(key); + return NULL; + } + PyObject* capsule = PyDict_GetItem(dict, key); + if (capsule) { + Py_DECREF(key); + return (struct pymb_registry*) PyCapsule_GetPointer( + capsule, "pymetabind_registry"); + } + struct pymb_registry* registry; + registry = (struct pymb_registry*) calloc(1, sizeof(*registry)); + if (registry) { + pymb_list_init(®istry->frameworks); + pymb_list_init(®istry->bindings); + capsule = PyCapsule_New(registry, "pymetabind_registry", NULL); + int rv = capsule ? PyDict_SetItem(dict, key, capsule) : -1; + Py_XDECREF(capsule); + if (rv != 0) { + free(registry); + registry = NULL; + } + } else { + PyErr_NoMemory(); + } + Py_DECREF(key); + return registry; +} + +/* + * Add a new framework to the given registry. Makes calls to + * framework->add_foreign_framework() and framework->add_foreign_binding() + * for each existing framework/binding in the registry. + */ +PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, + struct pymb_framework* framework) { +#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030e0000 + assert(framework->bindings_usable_forever && + "Free-threaded removal of bindings requires PyUnstable_TryIncRef(), " + "which was added in CPython 3.14"); +#endif + pymb_lock_registry(registry); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + // Intern `abi_extra` strings so they can be compared by pointer + if (other->abi_extra && framework->abi_extra && + 0 == strcmp(other->abi_extra, framework->abi_extra)) { + framework->abi_extra = other->abi_extra; + break; + } + } + pymb_list_append(®istry->frameworks, &framework->hook); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + if (other != framework) { + other->add_foreign_framework(framework); + framework->add_foreign_framework(other); + } + } + PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { + if (binding->framework != framework && pymb_try_ref_binding(binding)) { + framework->add_foreign_binding(binding); + pymb_unref_binding(binding); + } + } + pymb_unlock_registry(registry); +} + +/* Add a new binding to the given registry */ +PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, + struct pymb_binding* binding) { +#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000 + PyUnstable_EnableTryIncRef((PyObject *) binding->pytype); +#endif + PyObject* capsule = PyCapsule_New(binding, "pymetabind_binding", NULL); + int rv = -1; + if (capsule) { + rv = PyObject_SetAttrString((PyObject *) binding->pytype, + "__pymetabind_binding__", capsule); + Py_DECREF(capsule); + } + if (rv != 0) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } + pymb_lock_registry(registry); + pymb_list_append(®istry->bindings, &binding->hook); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + if (other != binding->framework) { + other->add_foreign_binding(binding); + } + } + pymb_unlock_registry(registry); +} + +/* + * Remove a binding from the given registry. This must be called during + * deallocation of the `binding->pytype`, such that its reference count is + * zero but still accessible. Once this function returns, you can free the + * binding structure. + */ +PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, + struct pymb_binding* binding) { + pymb_lock_registry(registry); + pymb_list_unlink(&binding->hook); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + if (other != binding->framework) { + other->remove_foreign_binding(binding); + } + } + pymb_unlock_registry(registry); +} + +/* + * Increase the reference count of a binding. Return 1 if successful (you can + * use the binding and must call pymb_unref_binding() when done) or 0 if the + * binding is being removed and shouldn't be used. + */ +PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding) { +#if defined(Py_GIL_DISABLED) + if (!binding->framework->bindings_usable_forever) { +#if PY_VERSION_HEX >= 0x030e0000 + return PyUnstable_TryIncRef((PyObject *) binding->pytype); +#else + // bindings_usable_forever is required on this Python version, and + // was checked in pymb_add_framework() + assert(false); +#endif + } +#else + Py_INCREF((PyObject *) binding->pytype); +#endif + return 1; +} + +/* Decrease the reference count of a binding. */ +PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding) { +#if defined(Py_GIL_DISABLED) + if (!binding->framework->bindings_usable_forever) { +#if PY_VERSION_HEX >= 0x030e0000 + Py_DECREF((PyObject *) binding->pytype); +#else + // bindings_usable_forever is required on this Python version, and + // was checked in pymb_add_framework() + assert(false); +#endif + } +#else + Py_DECREF((PyObject *) binding->pytype); +#endif +} + +/* + * Return a pointer to a pymb_binding for the Python type `type`, or NULL if + * none exists. + */ +PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type) { + PyObject* capsule = PyObject_GetAttrString(type, "__pymetabind_binding__"); + if (capsule == NULL) { + PyErr_Clear(); + return NULL; + } + void* binding = PyCapsule_GetPointer(capsule, "pymetabind_binding"); + Py_DECREF(capsule); + if (!binding) { + PyErr_Clear(); + } + return (struct pymb_binding*) binding; +} + +#endif /* defined(PYMB_DECLS_ONLY) */ + +#if defined(__cplusplus) +} +#endif diff --git a/include/pybind11/detail/struct_smart_holder.h b/include/pybind11/detail/struct_smart_holder.h index 5b65b4a9b2..8830b21346 100644 --- a/include/pybind11/detail/struct_smart_holder.h +++ b/include/pybind11/detail/struct_smart_holder.h @@ -68,7 +68,7 @@ static constexpr bool type_has_shared_from_this(...) { return false; } // This overload uses SFINAE to skip enable_shared_from_this checks when the // base is inaccessible (e.g. private inheritance). template -static auto type_has_shared_from_this(const T *ptr) +static constexpr auto type_has_shared_from_this(const T *ptr) -> decltype(static_cast *>(ptr), true) { return true; } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 1b23c5c681..0f6a64bee2 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -19,6 +19,7 @@ #include "dynamic_raw_ptr_cast_if_possible.h" #include "internals.h" #include "typeid.h" +#include "pymetabind.h" #include "using_smart_holder.h" #include "value_and_holder.h" @@ -38,6 +39,12 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +// Forward declaration, implemented in foreign.h +void* try_foreign_bindings( + const std::type_info *type, + void* (*attempt)(void *closure, pymb_binding *binding), + void *closure); + /// A life support system for temporary objects created by `type_caster::load()`. /// Adding a patient will keep it alive up until the enclosing function returns. class loader_life_support { @@ -83,6 +90,14 @@ class loader_life_support { Py_INCREF(h.ptr()); } } + + static bool can_add_patient() { + return get_internals().loader_life_support_tls.get() != nullptr; + } + + const std::unordered_set& list_patients() const { + return keep_alive; + } }; // Gets the cache entry for the given type, creating it if necessary. The return value is the pair @@ -230,9 +245,28 @@ PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_index &tp, return nullptr; } -PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { - detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); +PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, + bool throw_if_missing, + bool foreign_ok) { + if (detail::type_info *type_info = get_type_info(tp)) { + return handle((PyObject *) type_info->type); + } + if (foreign_ok) { + auto &foreign_internals = detail::get_foreign_internals(); + if (foreign_internals.imported_any) { + handle ret = with_internals([&](internals&) { + auto range = foreign_internals.bindings.equal_range(tp); + if (range.first != range.second) + return handle((PyObject *) range.first->second->pytype); + return handle(); + }); + if (ret) + return ret; + } + } + if (throw_if_missing) + return handle((PyObject *) get_type_info(tp, true)->type); + return nullptr; } inline bool try_incref(PyObject *obj) { @@ -469,8 +503,10 @@ PYBIND11_NOINLINE void instance::deallocate_layout() { } } -PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp) { - handle type = detail::get_type_handle(tp, false); +PYBIND11_NOINLINE bool isinstance_generic(handle obj, + const std::type_info &tp, + bool foreign_ok) { + handle type = detail::get_type_handle(tp, false, foreign_ok); if (!type) { return false; } @@ -869,21 +905,157 @@ class type_caster_generic { bool load(handle src, bool convert) { return load_impl(src, convert); } - PYBIND11_NOINLINE static handle cast(const void *_src, + // Information about how cast() can obtain its source object + struct cast_sources { + // A type-erased pointer and the type it points to + struct source { + const void *obj; + const std::type_info *type; + }; + + // Use the given pointer with its compile-time type, possibly downcast + // via polymorphic_type_hook() + template cast_sources(const itype *ptr); + + // Use the given pointer and type + cast_sources(const source& orig) : original(orig) { result = resolve(); } + + // Use the given object and pybind11 type info. NB: if tinfo is null, + // this does not provide enough information to use a foreign type or + // to render a useful error message + cast_sources(const void *obj, const detail::type_info *tinfo) + : original{obj, tinfo ? tinfo->cpptype : nullptr}, + result{obj, tinfo} {} + + // The object passed to cast(), with its static type. + // original.type must not be null if resolve() will be called. + // original.obj may be null if we're converting nullptr to a Python None + source original; + + // A more-derived version of `original` provided by a + // polymorphic_type_hook. downcast.type may be null if this is not + // a relevant concept for the current cast. + source downcast{}; + + // The source to use for this cast, and the corresponding pybind11 + // type_info. If the type_info is null, then pybind11 doesn't know + // about this type, but a foreign cast might work. + std::pair result; + + // cast() sets this to the foreign framework used, if any + mutable pymb_framework *used_foreign = nullptr; + + // Returns true if the cast will not succeed using a type known to + // pybind11 (it will either use a foreign-framework type, or fail). + bool needs_foreign() const { return result.second == nullptr; } + + // Returns true if the cast will use a pybind11 type that uses + // a smart holder. + bool creates_smart_holder() const { + return result.second != nullptr && + result.second->holder_enum_v == detail::holder_enum_t::smart_holder; + } + + private: + PYBIND11_NOINLINE std::pair resolve() { + if (downcast.type) { + if (same_type(*original.type, *downcast.type)) + const_cast(downcast.type) = nullptr; + else if (const auto *tpi = get_type_info(*downcast.type)) + return {downcast.obj, tpi}; + } + if (const auto *tpi = get_type_info(*original.type)) + return {original.obj, tpi}; + return {nullptr, nullptr}; + } + }; + + static handle cast(const void *src, + return_value_policy policy, + handle parent, + const detail::type_info *tinfo, + copy_or_move_ctor copy_constructor, + copy_or_move_ctor move_constructor, + const void *existing_holder = nullptr) { + cast_sources srcs{src, tinfo}; + return cast(srcs, policy, parent, copy_constructor, move_constructor, + existing_holder); + } + + static handle cast_foreign(const cast_sources &srcs, + return_value_policy policy, + handle parent) { + struct capture { + const void *src; + pymb_rv_policy policy; + PyObject *parent; + pymb_framework **used_foreign; + } cap; + + auto attempt = +[](void *closure, pymb_binding *binding) -> void* { + capture &cap = *(capture *) closure; + void *ret = binding->framework->to_python( + binding, const_cast(cap.src), + cap.policy, cap.parent); + if (ret) + *cap.used_foreign = binding->framework; + return ret; + }; + + cap.parent = parent.ptr(); + cap.used_foreign = &srcs.used_foreign; + switch (policy) { + case return_value_policy::automatic: + cap.policy = pymb_rv_policy_take_ownership; + break; + case return_value_policy::automatic_reference: + cap.policy = pymb_rv_policy_reference; + break; + default: + cap.policy = (pymb_rv_policy) (uint8_t) policy; + break; + } + if (srcs.downcast.type) { + cap.src = srcs.downcast.obj; + if (void *result = try_foreign_bindings(srcs.downcast.type, + attempt, &cap)) + return (PyObject *) result; + } + cap.src = srcs.original.obj; + if (void *result = try_foreign_bindings(srcs.original.type, + attempt, &cap)) + return (PyObject *) result; + return nullptr; + } + + PYBIND11_NOINLINE static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent, - const detail::type_info *tinfo, - void *(*copy_constructor)(const void *), - void *(*move_constructor)(const void *), + copy_or_move_ctor copy_constructor, + copy_or_move_ctor move_constructor, const void *existing_holder = nullptr) { - if (!tinfo) { // no type info: error will be set already + if (!srcs.result.second) { + // No pybind11 type info. See if we can use another framework's + // type to complete this cast. Set srcs.used_foreign if so. + if (get_foreign_internals().imported_any) { + if (handle ret = cast_foreign(srcs, policy, parent)) + return ret; + } + std::string tname = + srcs.downcast.type ? srcs.downcast.type->name() : + srcs.original.type ? srcs.original.type->name() : + ""; + detail::clean_type_id(tname); + std::string msg = "Unregistered type : " + tname; + set_error(PyExc_TypeError, msg.c_str()); return handle(); } - void *src = const_cast(_src); + void *src = const_cast(srcs.result.first); if (src == nullptr) { return none().release(); } + const type_info *tinfo = srcs.result.second; if (handle registered_inst = find_registered_python_instance(src, tinfo)) { return registered_inst; @@ -1018,6 +1190,7 @@ class type_caster_generic { return false; } void check_holder_compat() {} + bool set_foreign_holder(handle) { return true; } PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { auto caster = type_caster_generic(ti); @@ -1027,13 +1200,44 @@ class type_caster_generic { return nullptr; } + /// Try to load as a type exposed by a different binding framework. + bool try_load_other_framework(handle src, bool convert) { + auto& foreign_internals = get_foreign_internals(); + if (!foreign_internals.imported_any || !cpptype || src.is_none()) + return false; + + struct capture { + handle src; + bool convert; + } cap{src, convert}; + + auto attempt = +[](void *closure, pymb_binding *binding) -> void* { + capture &cap = *(capture *) closure; + void (*keep_referenced)(void*, PyObject*) = nullptr; + if (loader_life_support::can_add_patient()) { + keep_referenced = [](void *, PyObject *item) { + loader_life_support::add_patient(item); + }; + } + return binding->framework->from_python( + binding, cap.src.ptr(), cap.convert, + keep_referenced, nullptr); + }; + + if (void *result = try_foreign_bindings(cpptype, attempt, &cap)) { + value = result; + return true; + } + return false; + } + /// Try to load with foreign typeinfo, if available. Used when there is no /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { + PYBIND11_NOINLINE bool try_load_foreign(handle src, bool convert) { constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; const auto pytype = type::handle_of(src); if (!hasattr(pytype, local_key)) { - return false; + return try_load_other_framework(src, convert); } type_info *foreign_typeinfo = reinterpret_borrow(getattr(pytype, local_key)); @@ -1056,14 +1260,15 @@ class type_caster_generic { // logic (without having to resort to virtual inheritance). template PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { + auto &this_ = static_cast(*this); if (!src) { return false; } if (!typeinfo) { - return try_load_foreign_module_local(src); + return try_load_foreign(src, convert) && + this_.set_foreign_holder(src); } - auto &this_ = static_cast(*this); this_.check_holder_compat(); PyTypeObject *srctype = Py_TYPE(src.ptr()); @@ -1133,8 +1338,9 @@ class type_caster_generic { } } - // Global typeinfo has precedence over foreign module_local - if (try_load_foreign_module_local(src)) { + // Global typeinfo has precedence over foreign module_local and + // foreign frameworks + if (try_load_foreign(src, convert)) { return true; } @@ -1155,25 +1361,6 @@ class type_caster_generic { return false; } - // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast - // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair - // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). - PYBIND11_NOINLINE static std::pair - src_and_type(const void *src, - const std::type_info &cast_type, - const std::type_info *rtti_type = nullptr) { - if (auto *tpi = get_type_info(cast_type)) { - return {src, const_cast(tpi)}; - } - - // Not found, set error: - std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); - detail::clean_type_id(tname); - std::string msg = "Unregistered type : " + tname; - set_error(PyExc_TypeError, msg.c_str()); - return {nullptr, nullptr}; - } - const type_info *typeinfo = nullptr; const std::type_info *cpptype = nullptr; void *value = nullptr; @@ -1468,6 +1655,21 @@ struct polymorphic_type_hook : public polymorphic_type_hook_base {}; PYBIND11_NAMESPACE_BEGIN(detail) +template +type_caster_generic::cast_sources::cast_sources(const itype *ptr) + : original{ptr, &typeid(itype)} { + // If this is a base pointer to a derived type, and the derived type is + // registered with pybind11, we want to make the full derived object + // available. In the typical case where itype is polymorphic, we get the + // correct derived pointer (which may be != base pointer) by a dynamic_cast + // to most derived type. If itype is not polymorphic, a user-provided + // specialization of polymorphic_type_hook can do the same thing. + // If there is no downcast to perform, then the default hook will leave + // derived.type set to nullptr, which causes us to ignore derived.obj. + downcast.obj = polymorphic_type_hook::get(ptr, downcast.type); + result = resolve(); +} + /// Generic type caster for objects stored on the heap template class type_caster_base : public type_caster_generic { @@ -1479,6 +1681,10 @@ class type_caster_base : public type_caster_generic { type_caster_base() : type_caster_base(typeid(type)) {} explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) {} + struct cast_sources : type_caster_generic::cast_sources { + cast_sources(const itype *ptr) : type_caster_generic::cast_sources(ptr) {} + }; + static handle cast(const itype &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { @@ -1491,50 +1697,72 @@ class type_caster_base : public type_caster_generic { return cast(std::addressof(src), return_value_policy::move, parent); } - // Returns a (pointer, type_info) pair taking care of necessary type lookup for a - // polymorphic type (using RTTI by default, but can be overridden by specializing - // polymorphic_type_hook). If the instance isn't derived, returns the base version. - static std::pair src_and_type(const itype *src) { - const auto &cast_type = typeid(itype); - const std::type_info *instance_type = nullptr; - const void *vsrc = polymorphic_type_hook::get(src, instance_type); - if (instance_type && !same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type. If the derived type is registered - // with pybind11, we want to make the full derived object available. - // In the typical case where itype is polymorphic, we get the correct - // derived pointer (which may be != base pointer) by a dynamic_cast to - // most derived type. If itype is not polymorphic, we won't get here - // except via a user-provided specialization of polymorphic_type_hook, - // and the user has promised that no this-pointer adjustment is - // required in that case, so it's OK to use static_cast. - if (const auto *tpi = get_type_info(*instance_type)) { - return {vsrc, tpi}; - } - } - // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, - // so don't do a cast - return type_caster_generic::src_and_type(src, cast_type, instance_type); - } - - static handle cast(const itype *src, return_value_policy policy, handle parent) { - auto st = src_and_type(src); - return type_caster_generic::cast(st.first, - policy, - parent, - st.second, - make_copy_constructor(src), - make_move_constructor(src)); + static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent) { + return type_caster_generic::cast( + srcs, policy, parent, + make_copy_constructor((const itype *) nullptr), + make_move_constructor((const itype *) nullptr)); } - static handle cast_holder(const itype *src, const void *holder) { - auto st = src_and_type(src); - return type_caster_generic::cast(st.first, - return_value_policy::take_ownership, - {}, - st.second, - nullptr, - nullptr, - holder); + template + static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { + handle ret = type_caster_generic::cast( + srcs, return_value_policy::take_ownership, {}, + nullptr, nullptr, holder); + if (srcs.used_foreign) { + // Foreign cast succeeded; release C++ ownership + holder->release(); + } + return ret; + } + + PYBIND11_NOINLINE static void after_shared_ptr_cast_to_foreign( + handle ret, + std::shared_ptr holder, + pymb_framework *framework) { + // Make the resulting Python object keep a shared_ptr alive, + // even if there's not space for it inside the object. + auto sp = std::make_unique>(std::move(holder)); + if (-1 == framework->keep_alive( + ret.ptr(), sp.get(), [](void *p) noexcept { + delete (std::shared_ptr *) p; + })) { + ret.dec_ref(); + throw error_already_set(); + } + sp.release(); + } + + template + static handle cast_holder(const cast_sources &srcs, const std::shared_ptr *holder) { + // Use reference policy if casting via a foreign binding, and + // take_ownership if casting a pybind11 type + auto policy = srcs.needs_foreign() ? return_value_policy::reference : + return_value_policy::take_ownership; + handle ret = type_caster_generic::cast( + srcs, policy, {}, nullptr, nullptr, holder); + if (srcs.used_foreign) { + after_shared_ptr_cast_to_foreign( + ret, std::static_pointer_cast(*holder), + srcs.used_foreign); + } + return ret; + } + + static handle cast_holder(const cast_sources &srcs, const void *holder) { + auto policy = srcs.needs_foreign() ? return_value_policy::reference : + return_value_policy::take_ownership; + handle ret = type_caster_generic::cast( + srcs, policy, {}, nullptr, nullptr, holder); + if (srcs.used_foreign) { + PyErr_SetString(PyExc_TypeError, + "Can't cast foreign type to holder; only " + "returns of std::unique_ptr and std::shared_ptr " + "are supported for foreign types"); + ret.dec_ref(); + return nullptr; + } + return ret; } template @@ -1551,27 +1779,31 @@ class type_caster_base : public type_caster_generic { } protected: - using Constructor = void *(*) (const void *); - /* Only enabled when the types are {copy,move}-constructible *and* when the type does not have a private operator new implementation. A comma operator is used in the decltype argument to apply SFINAE to the public copy/move constructors.*/ template ::value>> static auto make_copy_constructor(const T *) - -> decltype(new T(std::declval()), Constructor{}) { + -> decltype(new T(std::declval()), copy_or_move_ctor{}) { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } template ::value>> static auto make_move_constructor(const T *) - -> decltype(new T(std::declval()), Constructor{}) { + -> decltype(new T(std::declval()), copy_or_move_ctor{}) { return [](const void *arg) -> void * { return new T(std::move(*const_cast(reinterpret_cast(arg)))); }; } - static Constructor make_copy_constructor(...) { return nullptr; } - static Constructor make_move_constructor(...) { return nullptr; } + static copy_or_move_ctor make_copy_constructor(...) { return nullptr; } + static copy_or_move_ctor make_move_constructor(...) { return nullptr; } + +public: + static std::pair copy_and_move_ctors() { + return {make_copy_constructor(static_cast(nullptr)), + make_move_constructor(static_cast(nullptr))}; + } }; inline std::string quote_cpp_type_name(const std::string &cpp_type_name) { @@ -1579,8 +1811,7 @@ inline std::string quote_cpp_type_name(const std::string &cpp_type_name) { } PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { - if (auto *type_data = get_type_info(ti)) { - handle th((PyObject *) type_data->type); + if (handle th = get_type_handle(ti, false, true)) { return th.attr("__module__").cast() + '.' + th.attr("__qualname__").cast(); } diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index a820bfbfc3..830dccb10b 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -245,6 +245,7 @@ inline void finalize_interpreter() { if (detail::get_num_interpreters_seen() > 1) { detail::get_internals_pp_manager().unref(); detail::get_local_internals_pp_manager().unref(); + detail::get_foreign_internals_pp_manager().unref(); // We know there can be no other interpreter alive now, so we can lower the count detail::get_num_interpreters_seen() = 1; @@ -256,14 +257,16 @@ inline void finalize_interpreter() { // and check it after Py_Finalize(). detail::get_internals_pp_manager().get_pp(); detail::get_local_internals_pp_manager().get_pp(); + detail::get_foreign_internals_pp_manager().get_pp(); Py_Finalize(); + // Internals contain data managed by the current interpreter, so we must + // clear them to avoid undefined behaviors when initializing another + // interpreter detail::get_internals_pp_manager().destroy(); - - // Local internals contains data managed by the current interpreter, so we must clear them to - // avoid undefined behaviors when initializing another interpreter detail::get_local_internals_pp_manager().destroy(); + detail::get_foreign_internals_pp_manager().destroy(); // We know there is no interpreter alive now, so we can reset the count detail::get_num_interpreters_seen() = 0; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 76519ad2f7..337deb5068 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -12,6 +12,7 @@ #include "detail/class.h" #include "detail/dynamic_raw_ptr_cast_if_possible.h" #include "detail/exception_translation.h" +#include "detail/foreign.h" #include "detail/function_record_pyobject.h" #include "detail/init.h" #include "detail/native_enum_data.h" @@ -157,8 +158,7 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel if (!t) { pybind11_fail("Internal error while parsing type signature (1)"); } - if (auto *tinfo = detail::get_type_info(*t)) { - handle th((PyObject *) tinfo->type); + if (handle th = detail::get_type_handle(*t, false, true)) { signature += th.attr("__module__").cast() + "." + th.attr("__qualname__").cast(); } else if (auto th = detail::global_internals_native_enum_type_map_get_item(*t)) { @@ -1652,6 +1652,11 @@ class generic_type : public object { #endif internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; PYBIND11_WARNING_POP + + auto &foreign_internals = get_foreign_internals(); + if (foreign_internals.export_all) { + foreign_internals.export_type_to_foreign(tinfo); + } }); if (rec.bases.size() > 1 || rec.multiple_inheritance) { @@ -2135,14 +2140,17 @@ class class_ : public detail::generic_type { generic_type::initialize(record); - if (has_alias) { - with_internals([&](internals &internals) { + with_internals([&](internals &internals) { + get_foreign_internals().copy_move_ctors.emplace( + *record.type, + detail::type_caster_base::copy_and_move_ctors()); + if (has_alias) { auto &instances = record.module_local ? get_local_internals().registered_types_cpp : internals.registered_types_cpp; instances[std::type_index(typeid(type_alias))] = instances[std::type_index(typeid(type))]; - }); - } + } + }); def("_pybind11_conduit_v1_", cpp_conduit_method); } @@ -2930,6 +2938,17 @@ PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { * internal list. */ add_patient(nurse.ptr(), patient.ptr()); } else { + if (Py_TYPE(nurse.ptr())->tp_weaklistoffset == 0) { + // The nurse type is not weak-referenceable. Maybe it is a + // different framework's type; try to get them to do the keep_alive. + if (auto *binding = pymb_get_binding(type::handle_of(nurse).ptr())) + if (0 != binding->framework->keep_alive(nurse.ptr(), + patient.ptr(), + nullptr)) + throw error_already_set(); + // Otherwise continue with the logic below (which will + // raise an error). + } /* Fall back to clever approach based on weak references taken from * Boost.Python. This is not used for pybind-registered types because * the objects can be destroyed out-of-order in a GC pass. */ diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 9c60c94c04..e4f4db5b8a 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -46,7 +46,7 @@ struct arg_v; PYBIND11_NAMESPACE_BEGIN(detail) class args_proxy; -bool isinstance_generic(handle obj, const std::type_info &tp); +bool isinstance_generic(handle obj, const std::type_info &tp, bool foreign_ok); template bool isinstance_native_enum(handle obj, const std::type_info &tp); @@ -855,13 +855,13 @@ bool isinstance(handle obj) { } template ::value, int> = 0> -bool isinstance(handle obj) { +bool isinstance(handle obj, bool foreign_ok = false) { return detail::isinstance_native_enum(obj, typeid(T)) - || detail::isinstance_generic(obj, typeid(T)); + || detail::isinstance_generic(obj, typeid(T), foreign_ok); } template <> -inline bool isinstance(handle) = delete; +inline bool isinstance(handle, bool) = delete; template <> inline bool isinstance(handle obj) { return obj.ptr() != nullptr; @@ -1573,14 +1573,14 @@ class type : public object { /// standard types, like int, float. etc. yet. /// See https://github.com/pybind/pybind11/issues/2486 template - static handle handle_of(); + static handle handle_of(bool foreign_ok = false); /// Convert C++ type to type if previously registered. Does not convert /// standard types, like int, float. etc. yet. /// See https://github.com/pybind/pybind11/issues/2486 template - static type of() { - return type(type::handle_of(), borrowed_t{}); + static type of(bool foreign_ok = false) { + return type(type::handle_of(foreign_ok), borrowed_t{}); } }; diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 4b208ed9b6..f0b8cb367b 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -180,6 +180,7 @@ class subinterpreter { // internals themselves. detail::get_internals_pp_manager().get_pp(); detail::get_local_internals_pp_manager().get_pp(); + detail::get_foreign_internals_pp_manager().get_pp(); // End it Py_EndInterpreter(destroy_tstate); @@ -188,6 +189,7 @@ class subinterpreter { // py::capsule calls `get_internals()` during destruction), so we destroy afterward. detail::get_internals_pp_manager().destroy(); detail::get_local_internals_pp_manager().destroy(); + detail::get_foreign_internals_pp_manager().destroy(); // switch back to the old tstate and old GIL (if there was one) if (switch_back) diff --git a/tests/test_class_sh_property_non_owning.cpp b/tests/test_class_sh_property_non_owning.cpp index 45fe7c7beb..c1cef41934 100644 --- a/tests/test_class_sh_property_non_owning.cpp +++ b/tests/test_class_sh_property_non_owning.cpp @@ -33,6 +33,8 @@ struct DataFieldsHolder { } } + DataFieldsHolder(DataFieldsHolder&&) noexcept = default; + DataField *vec_at(std::size_t index) { if (index >= vec.size()) { return nullptr; diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 627ccc591d..89d697c715 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -92,6 +92,8 @@ def test_independent_subinterpreters(): pytest.skip("Does not have subinterpreter support compiled in") code = """ +import sys +sys.path.append('.') import mod_per_interpreter_gil as m import pickle with open(pipeo, 'wb') as f: @@ -100,7 +102,10 @@ def test_independent_subinterpreters(): with create() as interp1, create() as interp2: try: - res0 = run_string(interp1, "import mod_shared_interpreter_gil") + res0 = run_string( + interp1, + "import sys; sys.path.append('.'); import mod_shared_interpreter_gil", + ) if res0 is not None: res0 = str(res0) except Exception as e: @@ -141,6 +146,8 @@ def test_independent_subinterpreters_modern(): from concurrent import interpreters code = """ +import sys +sys.path.append('.') import mod_per_interpreter_gil as m values.put_nowait(m.internals_at()) @@ -153,7 +160,9 @@ def test_independent_subinterpreters_modern(): interpreters.ExecutionFailed, match="does not support loading in subinterpreters", ): - interp1.exec("import mod_shared_interpreter_gil") + interp1.exec( + "import sys; sys.path.append('.'); import mod_shared_interpreter_gil" + ) values = interpreters.create_queue() interp1.prepare_main(values=values) @@ -185,6 +194,8 @@ def test_dependent_subinterpreters(): pytest.skip("Does not have subinterpreter support compiled in") code = """ +import sys +sys.path.append('.') import mod_shared_interpreter_gil as m import pickle with open(pipeo, 'wb') as f: From 1f47d3f7cf8ee4f001e7510271ff676358eec677 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:56:24 +0000 Subject: [PATCH 02/18] style: pre-commit fixes --- include/pybind11/cast.h | 12 +- include/pybind11/detail/foreign.h | 132 ++++++------- include/pybind11/detail/internals.h | 11 +- include/pybind11/detail/pymetabind.h | 197 ++++++++++---------- include/pybind11/detail/type_caster_base.h | 123 +++++------- include/pybind11/pybind11.h | 7 +- tests/test_class_sh_property_non_owning.cpp | 2 +- 7 files changed, 214 insertions(+), 270 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 3b439d5496..28d95349c2 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -893,8 +893,7 @@ struct holder_caster_foreign_helpers { } template - static bool set_foreign_holder(handle src, type *value, - std::shared_ptr *holder_out) { + static bool set_foreign_holder(handle src, type *value, std::shared_ptr *holder_out) { // We only support using std::shared_ptr for foreign T, and // it's done by creating a new shared_ptr control block that // owns a reference to the original Python object. @@ -910,8 +909,8 @@ struct holder_caster_foreign_helpers { } template - static bool set_foreign_holder(handle src, type *value, - std::shared_ptr *holder_out) { + static bool + set_foreign_holder(handle src, type *value, std::shared_ptr *holder_out) { std::shared_ptr holder_mut; if (set_foreign_holder(src, value, &holder_mut)) { *holder_out = holder_mut; @@ -972,8 +971,7 @@ struct copyable_holder_caster : public type_caster_base { } bool set_foreign_holder(handle src) { - return holder_caster_foreign_helpers::set_foreign_holder( - src, (type *) value, &holder); + return holder_caster_foreign_helpers::set_foreign_holder(src, (type *) value, &holder); } void load_value(value_and_holder &&v_h) { @@ -1109,7 +1107,7 @@ struct copyable_holder_caster< bool set_foreign_holder(handle src) { return holder_caster_foreign_helpers::set_foreign_holder( - src, (type *) value, &shared_ptr_storage); + src, (type *) value, &shared_ptr_storage); } void load_value(value_and_holder &&v_h) { diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index 9a24ad0ba2..c21357129e 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -11,15 +11,15 @@ #include "common.h" #include "internals.h" -#include "type_caster_base.h" #include "pymetabind.h" +#include "type_caster_base.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) // pybind11 exception translator that tries all known foreign ones PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { - auto& foreign_internals = get_foreign_internals(); + auto &foreign_internals = get_foreign_internals(); for (pymb_framework *fw : foreign_internals.exc_frameworks) { try { fw->translate_exception(&p); @@ -33,15 +33,13 @@ PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { // When learning about a new foreign type, should we automatically use it? inline bool should_autoimport_foreign(foreign_internals &foreign_internals, pymb_binding *binding) { - return foreign_internals.import_all && - binding->framework->abi_lang == pymb_abi_lang_cpp && - binding->framework->abi_extra == foreign_internals.self->abi_extra; + return foreign_internals.import_all && binding->framework->abi_lang == pymb_abi_lang_cpp + && binding->framework->abi_extra == foreign_internals.self->abi_extra; } // Add the given `binding` to our type maps so that we can use it to satisfy // from- and to-Python requests for the given C++ type -inline void import_foreign_binding(pymb_binding *binding, - const std::type_info *cpptype) noexcept { +inline void import_foreign_binding(pymb_binding *binding, const std::type_info *cpptype) noexcept { // Caller must hold the internals lock auto &foreign_internals = get_foreign_internals(); foreign_internals.imported_any = true; @@ -54,8 +52,7 @@ inline void import_foreign_binding(pymb_binding *binding, inline void *foreign_cb_from_python(pymb_binding *binding, PyObject *pyobj, uint8_t convert, - void (*keep_referenced)(void *ctx, - PyObject *obj), + void (*keep_referenced)(void *ctx, PyObject *obj), void *keep_referenced_ctx) noexcept { #if defined(PYBIND11_HAS_OPTIONAL) using maybe_life_support = std::optional; @@ -67,8 +64,8 @@ inline void *foreign_cb_from_python(pymb_binding *binding, bool engaged = false; maybe_life_support() {} - maybe_life_support(maybe_life_support&) = delete; - loader_life_support* operator->() { return &supp; } + maybe_life_support(maybe_life_support &) = delete; + loader_life_support *operator->() { return &supp; } void emplace() { new (&supp) loader_life_support(); engaged = true; @@ -84,8 +81,8 @@ inline void *foreign_cb_from_python(pymb_binding *binding, if (keep_referenced) { holder.emplace(); } - type_caster_generic caster{static_cast(binding->context)}; - void* ret = nullptr; + type_caster_generic caster{static_cast(binding->context)}; + void *ret = nullptr; try { if (caster.load(pyobj, convert)) { ret = caster.value; @@ -106,7 +103,7 @@ inline PyObject *foreign_cb_to_python(pymb_binding *binding, void *cobj, enum pymb_rv_policy rvp_, PyObject *parent) noexcept { - auto* ti = static_cast(binding->context); + auto *ti = static_cast(binding->context); if (cobj == nullptr) { return none().release().ptr(); } @@ -119,8 +116,8 @@ inline PyObject *foreign_cb_to_python(pymb_binding *binding, copy_or_move_ctor copy_ctor = nullptr, move_ctor = nullptr; if (rvp == return_value_policy::copy || rvp == return_value_policy::move) { - with_internals([&](internals&) { - auto& foreign_internals = get_foreign_internals(); + with_internals([&](internals &) { + auto &foreign_internals = get_foreign_internals(); auto it = foreign_internals.copy_move_ctors.find(*ti->cpptype); if (it != foreign_internals.copy_move_ctors.end()) { std::tie(copy_ctor, move_ctor) = it->second; @@ -129,20 +126,17 @@ inline PyObject *foreign_cb_to_python(pymb_binding *binding, } try { - return type_caster_generic::cast(cobj, rvp, parent, ti, - copy_ctor, move_ctor).ptr(); + return type_caster_generic::cast(cobj, rvp, parent, ti, copy_ctor, move_ctor).ptr(); } catch (...) { translate_exception(std::current_exception()); return nullptr; } } -inline int foreign_cb_keep_alive(PyObject *nurse, - void *payload, - void (*cb)(void*)) noexcept { +inline int foreign_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void *)) noexcept { try { if (!cb) { - keep_alive_impl(nurse, static_cast(payload)); + keep_alive_impl(nurse, static_cast(payload)); } else { capsule patient{payload, cb}; keep_alive_impl(nurse, patient); @@ -204,8 +198,8 @@ inline void foreign_cb_translate_exception(const void *eptr) { return; } catch (const builtin_exception &err) { // Could not use template since it's an abstract class. - if (const auto *nep = dynamic_cast( - std::addressof(err))) { + if (const auto *nep + = dynamic_cast(std::addressof(err))) { handle_nested_exception(*nep, e); } err.set_error(); @@ -216,18 +210,17 @@ inline void foreign_cb_translate_exception(const void *eptr) { } inline void foreign_cb_add_foreign_binding(pymb_binding *binding) noexcept { - with_internals([&](internals&) { - auto& foreign_internals = get_foreign_internals(); + with_internals([&](internals &) { + auto &foreign_internals = get_foreign_internals(); if (should_autoimport_foreign(foreign_internals, binding)) { - import_foreign_binding( - binding, (const std::type_info *) binding->native_type); + import_foreign_binding(binding, (const std::type_info *) binding->native_type); } }); } inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { - with_internals([&](internals&) { - auto& foreign_internals = get_foreign_internals(); + with_internals([&](internals &) { + auto &foreign_internals = get_foreign_internals(); auto remove_from_type = [&](const std::type_info *type) { auto range = foreign_internals.bindings.equal_range(*type); for (auto it = range.first; it != range.second; ++it) { @@ -237,8 +230,7 @@ inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { } } }; - bool should_remove_auto = - should_autoimport_foreign(foreign_internals, binding); + bool should_remove_auto = should_autoimport_foreign(foreign_internals, binding); auto it = foreign_internals.manual_imports.find(binding); if (it != foreign_internals.manual_imports.end()) { remove_from_type(it->second); @@ -250,13 +242,12 @@ inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { }); } -inline void foreign_cb_add_foreign_framework(pymb_framework *framework) - noexcept { +inline void foreign_cb_add_foreign_framework(pymb_framework *framework) noexcept { if (framework->translate_exception) { with_exception_translators( [&](std::forward_list &exception_translators, std::forward_list &) { - auto& foreign_internals = get_foreign_internals(); + auto &foreign_internals = get_foreign_internals(); if (foreign_internals.exc_frameworks.empty()) { // First foreign framework with an exception translator. // Add our `foreign_exception_translator` wrapper in the @@ -266,16 +257,14 @@ inline void foreign_cb_add_foreign_framework(pymb_framework *framework) auto trailer = exception_translators.before_begin(); while (++leader != exception_translators.end()) ++trailer; - exception_translators.insert_after( - trailer, foreign_exception_translator); + exception_translators.insert_after(trailer, foreign_exception_translator); } // Add the new framework at the end of the list auto leader = foreign_internals.exc_frameworks.begin(); auto trailer = leader; while (++leader != foreign_internals.exc_frameworks.end()) ++trailer; - foreign_internals.exc_frameworks.insert_after( - trailer, framework); + foreign_internals.exc_frameworks.insert_after(trailer, framework); }); } } @@ -284,7 +273,7 @@ inline void foreign_cb_add_foreign_framework(pymb_framework *framework) // Advertise our existence, and the above callbacks, to other frameworks PYBIND11_NOINLINE bool foreign_internals::initialize() { - bool inited_by_us = with_internals([&](internals&) { + bool inited_by_us = with_internals([&](internals &) { if (registry) return false; registry = pymb_get_registry(); @@ -323,10 +312,9 @@ inline foreign_internals::~foreign_internals() = default; // the C++ type by looking at the binding, and require that its ABI match ours. // Throws an exception on failure. Caller must hold the internals lock and have // already called foreign_internals.initialize_if_needed(). -PYBIND11_NOINLINE void import_foreign_type(type pytype, - const std::type_info *cpptype) { +PYBIND11_NOINLINE void import_foreign_type(type pytype, const std::type_info *cpptype) { auto &foreign_internals = get_foreign_internals(); - pymb_binding* binding = pymb_get_binding(pytype.ptr()); + pymb_binding *binding = pymb_get_binding(pytype.ptr()); if (!binding) pybind11_fail("pybind11::import_foreign_type(): type does not define " "a __pymetabind_binding__"); @@ -356,8 +344,8 @@ PYBIND11_NOINLINE void import_foreign_type(type pytype, // other C++ binding frameworks used by extension modules loaded in this // interpreter, both those that exist now and those bound in the future. PYBIND11_NOINLINE void foreign_enable_import_all() { - auto& foreign_internals = get_foreign_internals(); - bool proceed = with_internals([&](internals&) { + auto &foreign_internals = get_foreign_internals(); + bool proceed = with_internals([&](internals &) { if (foreign_internals.import_all) return false; foreign_internals.import_all = true; @@ -376,10 +364,8 @@ PYBIND11_NOINLINE void foreign_enable_import_all() { // self never change once they're non-null, so we can accesss them // without locking here. pymb_lock_registry(foreign_internals.registry); - PYMB_LIST_FOREACH(struct pymb_binding*, binding, - foreign_internals.registry->bindings) { - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding)) { + PYMB_LIST_FOREACH(struct pymb_binding *, binding, foreign_internals.registry->bindings) { + if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) { foreign_cb_add_foreign_binding(binding); pymb_unref_binding(binding); } @@ -391,7 +377,7 @@ PYBIND11_NOINLINE void foreign_enable_import_all() { // type object. Caller must hold the internals lock and have already called // foreign_internals.initialize_if_needed(). PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { - auto& foreign_internals = get_foreign_internals(); + auto &foreign_internals = get_foreign_internals(); auto range = foreign_internals.bindings.equal_range(*ti->cpptype); for (auto it = range.first; it != range.second; ++it) if (it->second->framework == foreign_internals.self.get()) @@ -407,7 +393,7 @@ PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { capsule tie_lifetimes((void *) binding, [](void *p) { pymb_binding *binding = (pymb_binding *) p; pymb_remove_binding(get_foreign_internals().registry, binding); - free(const_cast(binding->source_name)); + free(const_cast(binding->source_name)); delete binding; }); keep_alive_impl((PyObject *) ti->type, tie_lifetimes); @@ -419,20 +405,19 @@ PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { // Call `export_type_to_foreign()` for each type that currently exists in our // internals structure and each type created in the future. PYBIND11_NOINLINE void foreign_enable_export_all() { - auto& foreign_internals = get_foreign_internals(); - bool proceed = with_internals([&](internals&) { + auto &foreign_internals = get_foreign_internals(); + bool proceed = with_internals([&](internals &) { if (foreign_internals.export_all) return false; foreign_internals.export_all = true; - foreign_internals.export_type_to_foreign = - &detail::export_type_to_foreign; + foreign_internals.export_type_to_foreign = &detail::export_type_to_foreign; return true; }); if (!proceed) return; foreign_internals.initialize_if_needed(); - with_internals([&](internals& internals) { - for (const auto& entry : internals.registered_types_cpp) { + with_internals([&](internals &internals) { + for (const auto &entry : internals.registered_types_cpp) { detail::export_type_to_foreign(entry.second); } }); @@ -441,10 +426,10 @@ PYBIND11_NOINLINE void foreign_enable_export_all() { // Invoke `attempt(closure, binding)` for each foreign binding `binding` // that claims `type` and was not supplied by us, until one of them returns // non-null. Return that first non-null value, or null if all attempts failed. -PYBIND11_NOINLINE void* try_foreign_bindings( - const std::type_info *type, - void* (*attempt)(void *closure, pymb_binding *binding), - void *closure) { +PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, + void *(*attempt)(void *closure, + pymb_binding *binding), + void *closure) { auto &internals = get_internals(); auto &foreign_internals = get_foreign_internals(); @@ -457,8 +442,7 @@ PYBIND11_NOINLINE void* try_foreign_bindings( if (std::next(range.first) == range.second) { // Single binding - check that it's not our own auto *binding = range.first->second; - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding)) { + if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) { #ifdef Py_GIL_DISABLED // attempt() might execute Python code; drop the internals lock // to avoid a deadlock @@ -475,8 +459,7 @@ PYBIND11_NOINLINE void* try_foreign_bindings( #ifndef Py_GIL_DISABLED for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding)) { + if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) { void *result = attempt(closure, binding); pymb_unref_binding(binding); if (result) @@ -496,16 +479,14 @@ PYBIND11_NOINLINE void* try_foreign_bindings( size_t len = (size_t) std::distance(range.first, range.second); // Allocate temporary storage for that many pointers - pymb_binding **scratch = - (pymb_binding **) alloca(len * sizeof(pymb_binding*)); + pymb_binding **scratch = (pymb_binding **) alloca(len * sizeof(pymb_binding *)); pymb_binding **scratch_tail = scratch; // Iterate again, taking out strong references and saving pointers to // our scratch storage for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding)) + if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) *scratch_tail++ = binding; } @@ -537,11 +518,10 @@ inline void set_foreign_type_defaults(bool export_all, bool import_all) { template inline void import_foreign_type(type pytype) { const std::type_info *cpptype = std::is_void::value ? nullptr : &typeid(T); - auto& foreign_internals = detail::get_foreign_internals(); + auto &foreign_internals = detail::get_foreign_internals(); foreign_internals.initialize_if_needed(); - detail::with_internals([&](detail::internals&) { - detail::import_foreign_type(pytype, cpptype); - }); + detail::with_internals( + [&](detail::internals &) { detail::import_foreign_type(pytype, cpptype); }); } inline void export_type_to_foreign(type ty) { @@ -549,11 +529,9 @@ inline void export_type_to_foreign(type ty) { if (!ti) pybind11_fail("pybind11::export_type_to_foreign: not a " "pybind11 registered type"); - auto& foreign_internals = detail::get_foreign_internals(); + auto &foreign_internals = detail::get_foreign_internals(); foreign_internals.initialize_if_needed(); - detail::with_internals([&](detail::internals&) { - detail::export_type_to_foreign(ti); - }); + detail::with_internals([&](detail::internals &) { detail::export_type_to_foreign(ti); }); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 77746f18c7..6d5bdde885 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -181,7 +181,8 @@ struct type_equal_to { template using type_map = std::unordered_map; template -using type_multimap = std::unordered_multimap; +using type_multimap + = std::unordered_multimap; struct override_hash { inline size_t operator()(const std::pair &v) const { @@ -313,8 +314,7 @@ struct foreign_internals { // For each C++ type with a native binding, store pointers to its // copy and move constructors. These would ideally move inside `type_info` // on an ABI bump. Protected by internals::mutex. - type_map> copy_move_ctors; + type_map> copy_move_ctors; // List of frameworks that provide exception translators. // Protected by internals::exception_translator_mutex. @@ -368,7 +368,7 @@ struct foreign_internals { return initialize(); } - private: +private: inline bool initialize(); }; @@ -425,7 +425,8 @@ struct type_info { bool module_local : 1; }; -#define PYBIND11_ABI_TAG "v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ +#define PYBIND11_ABI_TAG \ + "v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID #define PYBIND11_INTERNALS_ID "__pybind11_internals_" PYBIND11_ABI_TAG "__" diff --git a/include/pybind11/detail/pymetabind.h b/include/pybind11/detail/pymetabind.h index d42cd9c7eb..e026f01012 100644 --- a/include/pybind11/detail/pymetabind.h +++ b/include/pybind11/detail/pymetabind.h @@ -38,11 +38,11 @@ #pragma once -#include #include +#include #if !defined(PY_VERSION_HEX) -# error You must include Python.h before this header +# error You must include Python.h before this header #endif /* @@ -59,7 +59,7 @@ * compilation unit that doesn't request `PYMB_DECLS_ONLY`. */ #if !defined(PYMB_FUNC) -#define PYMB_FUNC inline +# define PYMB_FUNC inline #endif #if defined(__cplusplus) @@ -132,11 +132,11 @@ struct pymb_list { struct pymb_list_node head; }; -inline void pymb_list_init(struct pymb_list* list) { +inline void pymb_list_init(struct pymb_list *list) { list->head.prev = list->head.next = &list->head; } -inline void pymb_list_unlink(struct pymb_list_node* node) { +inline void pymb_list_unlink(struct pymb_list_node *node) { if (node->next) { node->next->prev = node->prev; node->prev->next = node->next; @@ -144,19 +144,17 @@ inline void pymb_list_unlink(struct pymb_list_node* node) { } } -inline void pymb_list_append(struct pymb_list* list, - struct pymb_list_node* node) { +inline void pymb_list_append(struct pymb_list *list, struct pymb_list_node *node) { pymb_list_unlink(node); - struct pymb_list_node* tail = list->head.prev; + struct pymb_list_node *tail = list->head.prev; tail->next = node; list->head.prev = node; node->prev = tail; node->next = &list->head; } -#define PYMB_LIST_FOREACH(type, name, list) \ - for (type name = (type) (list).head.next; \ - name != (type) &(list).head; \ +#define PYMB_LIST_FOREACH(type, name, list) \ + for (type name = (type) (list).head.next; name != (type) & (list).head; \ name = (type) name->hook.next) /* @@ -196,15 +194,13 @@ struct pymb_registry { }; #if defined(Py_GIL_DISALED) -inline void pymb_lock_registry(struct pymb_registry* registry) { - PyMutex_Lock(®istry->mutex); -} -inline void pymb_unlock_registry(struct pymb_registry* registry) { +inline void pymb_lock_registry(struct pymb_registry *registry) { PyMutex_Lock(®istry->mutex); } +inline void pymb_unlock_registry(struct pymb_registry *registry) { PyMutex_Unlock(®istry->mutex); } #else -inline void pymb_lock_registry(struct pymb_registry*) {} -inline void pymb_unlock_registry(struct pymb_registry*) {} +inline void pymb_lock_registry(struct pymb_registry *) {} +inline void pymb_unlock_registry(struct pymb_registry *) {} #endif struct pymb_binding; @@ -241,7 +237,7 @@ struct pymb_framework { struct pymb_list_node hook; // Human-readable description of this framework, as a NUL-terminated string - const char* name; + const char *name; // Does this framework guarantee that its `pymb_binding` structures remain // valid to use for the lifetime of the Python interpreter process once @@ -277,7 +273,7 @@ struct pymb_framework { // form of interning to speed up checking that a given binding is usable. // Thus, to check whether another framework's ABI matches yours, you can // do a pointer comparison `me->abi_extra == them->abi_extra`. - const char* abi_extra; + const char *abi_extra; // The function pointers below allow other frameworks to interact with // bindings provided by this framework. They are constant after construction @@ -307,11 +303,11 @@ struct pymb_framework { // On free-threaded builds, callers must ensure that the `binding` is not // destroyed during a call to `from_python`. The requirements for this are // subtle; see the full discussion in the comment for `struct pymb_binding`. - void* (*from_python)(struct pymb_binding* binding, - PyObject* pyobj, + void *(*from_python)(struct pymb_binding *binding, + PyObject *pyobj, uint8_t convert, - void (*keep_referenced)(void* ctx, PyObject* obj), - void* keep_referenced_ctx); + void (*keep_referenced)(void *ctx, PyObject *obj), + void *keep_referenced_ctx); // Wrap the C/C++/etc object `cobj` into a Python object using the given // return value policy. The type is specified by providing a `pymb_binding*` @@ -328,10 +324,10 @@ struct pymb_framework { // On free-threaded builds, callers must ensure that the `binding` is not // destroyed during a call to `to_python`. The requirements for this are // subtle; see the full discussion in the comment for `struct pymb_binding`. - PyObject* (*to_python)(struct pymb_binding* binding, - void* cobj, + PyObject *(*to_python)(struct pymb_binding *binding, + void *cobj, enum pymb_rv_policy rvp, - PyObject* parent); + PyObject *parent); // Request that a PyObject reference be dropped, or that a callback // be invoked, when `nurse` is destroyed. `nurse` should be an object @@ -341,7 +337,7 @@ struct pymb_framework { // or -1 and sets the Python error indicator on error. // // No synchronization is required to call this method. - int (*keep_alive)(PyObject* nurse, void* payload, void (*cb)(void*)); + int (*keep_alive)(PyObject *nurse, void *payload, void (*cb)(void *)); // Attempt to translate a C++ exception known to this framework to Python. // This should translate only framework-specific exceptions or user-defined @@ -353,28 +349,28 @@ struct pymb_framework { // C++ exception translation. // // No synchronization is required to call this method. - void (*translate_exception)(const void* eptr); + void (*translate_exception)(const void *eptr); // Notify this framework that some other framework published a new binding. // This call will be made after the new binding has been linked into the // `pymb_registry::bindings` list. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*add_foreign_binding)(struct pymb_binding* binding); + void (*add_foreign_binding)(struct pymb_binding *binding); // Notify this framework that some other framework is about to remove // a binding. This call will be made after the binding has been removed // from the `pymb_registry::bindings` list. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*remove_foreign_binding)(struct pymb_binding* binding); + void (*remove_foreign_binding)(struct pymb_binding *binding); // Notify this framework that some other framework came into existence. // This call will be made after the new framework has been linked into the // `pymb_registry::frameworks` list and before it adds any bindings. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*add_foreign_framework)(struct pymb_framework* framework); + void (*add_foreign_framework)(struct pymb_framework *framework); // There is no remove_foreign_framework(); the interpreter has // already been finalized at that point, so there's nothing for the @@ -477,26 +473,26 @@ struct pymb_binding { struct pymb_list_node hook; // The framework that provides this binding - struct pymb_framework* framework; + struct pymb_framework *framework; // Python type: you will get an instance of this type from a successful // call to `framework::from_python()` that passes this binding - PyTypeObject* pytype; + PyTypeObject *pytype; // The native identifier for this type in `framework->abi_lang`, if that is // a concept that exists in that language. See the documentation of // `enum pymb_abi_lang` for specific per-language semantics. - const void* native_type; + const void *native_type; // The way that this type would be written in `framework->abi_lang` source // code, as a NUL-terminated byte string without struct/class/enum words. // Examples: "Foo", "Bar::Baz", "std::vector >" - const char* source_name; + const char *source_name; // Pointer that is free for use by the framework, e.g., to point to its // own data about this type. If the framework needs more data, it can // over-allocate the `pymb_binding` storage and use the space after this. - void* context; + void *context; }; /* @@ -505,18 +501,16 @@ struct pymb_binding { * considered part of the ABI. */ -PYMB_FUNC struct pymb_registry* pymb_get_registry(); -PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, - struct pymb_framework* framework); -PYMB_FUNC void pymb_remove_framework(struct pymb_registry* registry, - struct pymb_framework* framework); -PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, - struct pymb_binding* binding); -PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, - struct pymb_binding* binding); -PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding); -PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding); -PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); +PYMB_FUNC struct pymb_registry *pymb_get_registry(); +PYMB_FUNC void pymb_add_framework(struct pymb_registry *registry, + struct pymb_framework *framework); +PYMB_FUNC void pymb_remove_framework(struct pymb_registry *registry, + struct pymb_framework *framework); +PYMB_FUNC void pymb_add_binding(struct pymb_registry *registry, struct pymb_binding *binding); +PYMB_FUNC void pymb_remove_binding(struct pymb_registry *registry, struct pymb_binding *binding); +PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding *binding); +PYMB_FUNC void pymb_unref_binding(struct pymb_binding *binding); +PYMB_FUNC struct pymb_binding *pymb_get_binding(PyObject *type); #if !defined(PYMB_DECLS_ONLY) @@ -526,27 +520,26 @@ PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); * This must be called from a module initialization function so that the * import lock can provide mutual exclusion. */ -PYMB_FUNC struct pymb_registry* pymb_get_registry() { -#if defined(PYPY_VERSION) - PyObject* dict = PyEval_GetBuiltins(); -#elif PY_VERSION_HEX < 0x03090000 - PyObject* dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); -#else - PyObject* dict = PyInterpreterState_GetDict(PyInterpreterState_Get()); -#endif - PyObject* key = PyUnicode_FromString("__pymetabind_registry__"); +PYMB_FUNC struct pymb_registry *pymb_get_registry() { +# if defined(PYPY_VERSION) + PyObject *dict = PyEval_GetBuiltins(); +# elif PY_VERSION_HEX < 0x03090000 + PyObject *dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); +# else + PyObject *dict = PyInterpreterState_GetDict(PyInterpreterState_Get()); +# endif + PyObject *key = PyUnicode_FromString("__pymetabind_registry__"); if (!dict || !key) { Py_XDECREF(key); return NULL; } - PyObject* capsule = PyDict_GetItem(dict, key); + PyObject *capsule = PyDict_GetItem(dict, key); if (capsule) { Py_DECREF(key); - return (struct pymb_registry*) PyCapsule_GetPointer( - capsule, "pymetabind_registry"); + return (struct pymb_registry *) PyCapsule_GetPointer(capsule, "pymetabind_registry"); } - struct pymb_registry* registry; - registry = (struct pymb_registry*) calloc(1, sizeof(*registry)); + struct pymb_registry *registry; + registry = (struct pymb_registry *) calloc(1, sizeof(*registry)); if (registry) { pymb_list_init(®istry->frameworks); pymb_list_init(®istry->bindings); @@ -569,30 +562,30 @@ PYMB_FUNC struct pymb_registry* pymb_get_registry() { * framework->add_foreign_framework() and framework->add_foreign_binding() * for each existing framework/binding in the registry. */ -PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, - struct pymb_framework* framework) { -#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030e0000 - assert(framework->bindings_usable_forever && - "Free-threaded removal of bindings requires PyUnstable_TryIncRef(), " - "which was added in CPython 3.14"); -#endif +PYMB_FUNC void pymb_add_framework(struct pymb_registry *registry, + struct pymb_framework *framework) { +# if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030e0000 + assert(framework->bindings_usable_forever + && "Free-threaded removal of bindings requires PyUnstable_TryIncRef(), " + "which was added in CPython 3.14"); +# endif pymb_lock_registry(registry); - PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { // Intern `abi_extra` strings so they can be compared by pointer - if (other->abi_extra && framework->abi_extra && - 0 == strcmp(other->abi_extra, framework->abi_extra)) { + if (other->abi_extra && framework->abi_extra + && 0 == strcmp(other->abi_extra, framework->abi_extra)) { framework->abi_extra = other->abi_extra; break; } } pymb_list_append(®istry->frameworks, &framework->hook); - PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { if (other != framework) { other->add_foreign_framework(framework); framework->add_foreign_framework(other); } } - PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { + PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { if (binding->framework != framework && pymb_try_ref_binding(binding)) { framework->add_foreign_binding(binding); pymb_unref_binding(binding); @@ -602,16 +595,15 @@ PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, } /* Add a new binding to the given registry */ -PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, - struct pymb_binding* binding) { -#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000 +PYMB_FUNC void pymb_add_binding(struct pymb_registry *registry, struct pymb_binding *binding) { +# if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000 PyUnstable_EnableTryIncRef((PyObject *) binding->pytype); -#endif - PyObject* capsule = PyCapsule_New(binding, "pymetabind_binding", NULL); +# endif + PyObject *capsule = PyCapsule_New(binding, "pymetabind_binding", NULL); int rv = -1; if (capsule) { - rv = PyObject_SetAttrString((PyObject *) binding->pytype, - "__pymetabind_binding__", capsule); + rv = PyObject_SetAttrString( + (PyObject *) binding->pytype, "__pymetabind_binding__", capsule); Py_DECREF(capsule); } if (rv != 0) { @@ -619,7 +611,7 @@ PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, } pymb_lock_registry(registry); pymb_list_append(®istry->bindings, &binding->hook); - PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { if (other != binding->framework) { other->add_foreign_binding(binding); } @@ -633,11 +625,10 @@ PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, * zero but still accessible. Once this function returns, you can free the * binding structure. */ -PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, - struct pymb_binding* binding) { +PYMB_FUNC void pymb_remove_binding(struct pymb_registry *registry, struct pymb_binding *binding) { pymb_lock_registry(registry); pymb_list_unlink(&binding->hook); - PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { if (other != binding->framework) { other->remove_foreign_binding(binding); } @@ -650,56 +641,56 @@ PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, * use the binding and must call pymb_unref_binding() when done) or 0 if the * binding is being removed and shouldn't be used. */ -PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding) { -#if defined(Py_GIL_DISABLED) +PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding *binding) { +# if defined(Py_GIL_DISABLED) if (!binding->framework->bindings_usable_forever) { -#if PY_VERSION_HEX >= 0x030e0000 +# if PY_VERSION_HEX >= 0x030e0000 return PyUnstable_TryIncRef((PyObject *) binding->pytype); -#else +# else // bindings_usable_forever is required on this Python version, and // was checked in pymb_add_framework() assert(false); -#endif +# endif } -#else +# else Py_INCREF((PyObject *) binding->pytype); -#endif +# endif return 1; } /* Decrease the reference count of a binding. */ -PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding) { -#if defined(Py_GIL_DISABLED) +PYMB_FUNC void pymb_unref_binding(struct pymb_binding *binding) { +# if defined(Py_GIL_DISABLED) if (!binding->framework->bindings_usable_forever) { -#if PY_VERSION_HEX >= 0x030e0000 +# if PY_VERSION_HEX >= 0x030e0000 Py_DECREF((PyObject *) binding->pytype); -#else +# else // bindings_usable_forever is required on this Python version, and // was checked in pymb_add_framework() assert(false); -#endif +# endif } -#else +# else Py_DECREF((PyObject *) binding->pytype); -#endif +# endif } /* * Return a pointer to a pymb_binding for the Python type `type`, or NULL if * none exists. */ -PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type) { - PyObject* capsule = PyObject_GetAttrString(type, "__pymetabind_binding__"); +PYMB_FUNC struct pymb_binding *pymb_get_binding(PyObject *type) { + PyObject *capsule = PyObject_GetAttrString(type, "__pymetabind_binding__"); if (capsule == NULL) { PyErr_Clear(); return NULL; } - void* binding = PyCapsule_GetPointer(capsule, "pymetabind_binding"); + void *binding = PyCapsule_GetPointer(capsule, "pymetabind_binding"); Py_DECREF(capsule); if (!binding) { PyErr_Clear(); } - return (struct pymb_binding*) binding; + return (struct pymb_binding *) binding; } #endif /* defined(PYMB_DECLS_ONLY) */ diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 0f6a64bee2..773cf8ce14 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -18,8 +18,8 @@ #include "descr.h" #include "dynamic_raw_ptr_cast_if_possible.h" #include "internals.h" -#include "typeid.h" #include "pymetabind.h" +#include "typeid.h" #include "using_smart_holder.h" #include "value_and_holder.h" @@ -40,10 +40,9 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) // Forward declaration, implemented in foreign.h -void* try_foreign_bindings( - const std::type_info *type, - void* (*attempt)(void *closure, pymb_binding *binding), - void *closure); +void *try_foreign_bindings(const std::type_info *type, + void *(*attempt)(void *closure, pymb_binding *binding), + void *closure); /// A life support system for temporary objects created by `type_caster::load()`. /// Adding a patient will keep it alive up until the enclosing function returns. @@ -95,9 +94,7 @@ class loader_life_support { return get_internals().loader_life_support_tls.get() != nullptr; } - const std::unordered_set& list_patients() const { - return keep_alive; - } + const std::unordered_set &list_patients() const { return keep_alive; } }; // Gets the cache entry for the given type, creating it if necessary. The return value is the pair @@ -254,7 +251,7 @@ PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, if (foreign_ok) { auto &foreign_internals = detail::get_foreign_internals(); if (foreign_internals.imported_any) { - handle ret = with_internals([&](internals&) { + handle ret = with_internals([&](internals &) { auto range = foreign_internals.bindings.equal_range(tp); if (range.first != range.second) return handle((PyObject *) range.first->second->pytype); @@ -503,9 +500,7 @@ PYBIND11_NOINLINE void instance::deallocate_layout() { } } -PYBIND11_NOINLINE bool isinstance_generic(handle obj, - const std::type_info &tp, - bool foreign_ok) { +PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp, bool foreign_ok) { handle type = detail::get_type_handle(tp, false, foreign_ok); if (!type) { return false; @@ -915,17 +910,17 @@ class type_caster_generic { // Use the given pointer with its compile-time type, possibly downcast // via polymorphic_type_hook() - template cast_sources(const itype *ptr); + template + cast_sources(const itype *ptr); // Use the given pointer and type - cast_sources(const source& orig) : original(orig) { result = resolve(); } + cast_sources(const source &orig) : original(orig) { result = resolve(); } // Use the given object and pybind11 type info. NB: if tinfo is null, // this does not provide enough information to use a foreign type or // to render a useful error message cast_sources(const void *obj, const detail::type_info *tinfo) - : original{obj, tinfo ? tinfo->cpptype : nullptr}, - result{obj, tinfo} {} + : original{obj, tinfo ? tinfo->cpptype : nullptr}, result{obj, tinfo} {} // The object passed to cast(), with its static type. // original.type must not be null if resolve() will be called. @@ -952,11 +947,11 @@ class type_caster_generic { // Returns true if the cast will use a pybind11 type that uses // a smart holder. bool creates_smart_holder() const { - return result.second != nullptr && - result.second->holder_enum_v == detail::holder_enum_t::smart_holder; + return result.second != nullptr + && result.second->holder_enum_v == detail::holder_enum_t::smart_holder; } - private: + private: PYBIND11_NOINLINE std::pair resolve() { if (downcast.type) { if (same_type(*original.type, *downcast.type)) @@ -978,13 +973,11 @@ class type_caster_generic { copy_or_move_ctor move_constructor, const void *existing_holder = nullptr) { cast_sources srcs{src, tinfo}; - return cast(srcs, policy, parent, copy_constructor, move_constructor, - existing_holder); + return cast(srcs, policy, parent, copy_constructor, move_constructor, existing_holder); } - static handle cast_foreign(const cast_sources &srcs, - return_value_policy policy, - handle parent) { + static handle + cast_foreign(const cast_sources &srcs, return_value_policy policy, handle parent) { struct capture { const void *src; pymb_rv_policy policy; @@ -992,11 +985,10 @@ class type_caster_generic { pymb_framework **used_foreign; } cap; - auto attempt = +[](void *closure, pymb_binding *binding) -> void* { + auto attempt = +[](void *closure, pymb_binding *binding) -> void * { capture &cap = *(capture *) closure; void *ret = binding->framework->to_python( - binding, const_cast(cap.src), - cap.policy, cap.parent); + binding, const_cast(cap.src), cap.policy, cap.parent); if (ret) *cap.used_foreign = binding->framework; return ret; @@ -1017,13 +1009,11 @@ class type_caster_generic { } if (srcs.downcast.type) { cap.src = srcs.downcast.obj; - if (void *result = try_foreign_bindings(srcs.downcast.type, - attempt, &cap)) + if (void *result = try_foreign_bindings(srcs.downcast.type, attempt, &cap)) return (PyObject *) result; } cap.src = srcs.original.obj; - if (void *result = try_foreign_bindings(srcs.original.type, - attempt, &cap)) + if (void *result = try_foreign_bindings(srcs.original.type, attempt, &cap)) return (PyObject *) result; return nullptr; } @@ -1041,10 +1031,9 @@ class type_caster_generic { if (handle ret = cast_foreign(srcs, policy, parent)) return ret; } - std::string tname = - srcs.downcast.type ? srcs.downcast.type->name() : - srcs.original.type ? srcs.original.type->name() : - ""; + std::string tname = srcs.downcast.type ? srcs.downcast.type->name() + : srcs.original.type ? srcs.original.type->name() + : ""; detail::clean_type_id(tname); std::string msg = "Unregistered type : " + tname; set_error(PyExc_TypeError, msg.c_str()); @@ -1202,7 +1191,7 @@ class type_caster_generic { /// Try to load as a type exposed by a different binding framework. bool try_load_other_framework(handle src, bool convert) { - auto& foreign_internals = get_foreign_internals(); + auto &foreign_internals = get_foreign_internals(); if (!foreign_internals.imported_any || !cpptype || src.is_none()) return false; @@ -1211,17 +1200,15 @@ class type_caster_generic { bool convert; } cap{src, convert}; - auto attempt = +[](void *closure, pymb_binding *binding) -> void* { + auto attempt = +[](void *closure, pymb_binding *binding) -> void * { capture &cap = *(capture *) closure; - void (*keep_referenced)(void*, PyObject*) = nullptr; + void (*keep_referenced)(void *, PyObject *) = nullptr; if (loader_life_support::can_add_patient()) { - keep_referenced = [](void *, PyObject *item) { - loader_life_support::add_patient(item); - }; + keep_referenced + = [](void *, PyObject *item) { loader_life_support::add_patient(item); }; } return binding->framework->from_python( - binding, cap.src.ptr(), cap.convert, - keep_referenced, nullptr); + binding, cap.src.ptr(), cap.convert, keep_referenced, nullptr); }; if (void *result = try_foreign_bindings(cpptype, attempt, &cap)) { @@ -1265,8 +1252,7 @@ class type_caster_generic { return false; } if (!typeinfo) { - return try_load_foreign(src, convert) && - this_.set_foreign_holder(src); + return try_load_foreign(src, convert) && this_.set_foreign_holder(src); } this_.check_holder_compat(); @@ -1656,8 +1642,7 @@ struct polymorphic_type_hook : public polymorphic_type_hook_base {}; PYBIND11_NAMESPACE_BEGIN(detail) template -type_caster_generic::cast_sources::cast_sources(const itype *ptr) - : original{ptr, &typeid(itype)} { +type_caster_generic::cast_sources::cast_sources(const itype *ptr) : original{ptr, &typeid(itype)} { // If this is a base pointer to a derived type, and the derived type is // registered with pybind11, we want to make the full derived object // available. In the typical case where itype is polymorphic, we get the @@ -1698,17 +1683,17 @@ class type_caster_base : public type_caster_generic { } static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent) { - return type_caster_generic::cast( - srcs, policy, parent, - make_copy_constructor((const itype *) nullptr), - make_move_constructor((const itype *) nullptr)); + return type_caster_generic::cast(srcs, + policy, + parent, + make_copy_constructor((const itype *) nullptr), + make_move_constructor((const itype *) nullptr)); } template static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { handle ret = type_caster_generic::cast( - srcs, return_value_policy::take_ownership, {}, - nullptr, nullptr, holder); + srcs, return_value_policy::take_ownership, {}, nullptr, nullptr, holder); if (srcs.used_foreign) { // Foreign cast succeeded; release C++ ownership holder->release(); @@ -1717,16 +1702,13 @@ class type_caster_base : public type_caster_generic { } PYBIND11_NOINLINE static void after_shared_ptr_cast_to_foreign( - handle ret, - std::shared_ptr holder, - pymb_framework *framework) { + handle ret, std::shared_ptr holder, pymb_framework *framework) { // Make the resulting Python object keep a shared_ptr alive, // even if there's not space for it inside the object. auto sp = std::make_unique>(std::move(holder)); - if (-1 == framework->keep_alive( - ret.ptr(), sp.get(), [](void *p) noexcept { - delete (std::shared_ptr *) p; - })) { + if (-1 == framework->keep_alive(ret.ptr(), sp.get(), [](void *p) noexcept { + delete (std::shared_ptr *) p; + })) { ret.dec_ref(); throw error_already_set(); } @@ -1737,23 +1719,20 @@ class type_caster_base : public type_caster_generic { static handle cast_holder(const cast_sources &srcs, const std::shared_ptr *holder) { // Use reference policy if casting via a foreign binding, and // take_ownership if casting a pybind11 type - auto policy = srcs.needs_foreign() ? return_value_policy::reference : - return_value_policy::take_ownership; - handle ret = type_caster_generic::cast( - srcs, policy, {}, nullptr, nullptr, holder); + auto policy = srcs.needs_foreign() ? return_value_policy::reference + : return_value_policy::take_ownership; + handle ret = type_caster_generic::cast(srcs, policy, {}, nullptr, nullptr, holder); if (srcs.used_foreign) { after_shared_ptr_cast_to_foreign( - ret, std::static_pointer_cast(*holder), - srcs.used_foreign); + ret, std::static_pointer_cast(*holder), srcs.used_foreign); } return ret; } static handle cast_holder(const cast_sources &srcs, const void *holder) { - auto policy = srcs.needs_foreign() ? return_value_policy::reference : - return_value_policy::take_ownership; - handle ret = type_caster_generic::cast( - srcs, policy, {}, nullptr, nullptr, holder); + auto policy = srcs.needs_foreign() ? return_value_policy::reference + : return_value_policy::take_ownership; + handle ret = type_caster_generic::cast(srcs, policy, {}, nullptr, nullptr, holder); if (srcs.used_foreign) { PyErr_SetString(PyExc_TypeError, "Can't cast foreign type to holder; only " @@ -1801,8 +1780,8 @@ class type_caster_base : public type_caster_generic { public: static std::pair copy_and_move_ctors() { - return {make_copy_constructor(static_cast(nullptr)), - make_move_constructor(static_cast(nullptr))}; + return {make_copy_constructor(static_cast(nullptr)), + make_move_constructor(static_cast(nullptr))}; } }; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 337deb5068..548e5e1e38 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2142,8 +2142,7 @@ class class_ : public detail::generic_type { with_internals([&](internals &internals) { get_foreign_internals().copy_move_ctors.emplace( - *record.type, - detail::type_caster_base::copy_and_move_ctors()); + *record.type, detail::type_caster_base::copy_and_move_ctors()); if (has_alias) { auto &instances = record.module_local ? get_local_internals().registered_types_cpp : internals.registered_types_cpp; @@ -2942,9 +2941,7 @@ PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { // The nurse type is not weak-referenceable. Maybe it is a // different framework's type; try to get them to do the keep_alive. if (auto *binding = pymb_get_binding(type::handle_of(nurse).ptr())) - if (0 != binding->framework->keep_alive(nurse.ptr(), - patient.ptr(), - nullptr)) + if (0 != binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr)) throw error_already_set(); // Otherwise continue with the logic below (which will // raise an error). diff --git a/tests/test_class_sh_property_non_owning.cpp b/tests/test_class_sh_property_non_owning.cpp index c1cef41934..5a5877e4e6 100644 --- a/tests/test_class_sh_property_non_owning.cpp +++ b/tests/test_class_sh_property_non_owning.cpp @@ -33,7 +33,7 @@ struct DataFieldsHolder { } } - DataFieldsHolder(DataFieldsHolder&&) noexcept = default; + DataFieldsHolder(DataFieldsHolder &&) noexcept = default; DataField *vec_at(std::size_t index) { if (index >= vec.size()) { From 493a98ea55f2cb6ca581cfd6ffd196ad39249fd5 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sun, 17 Aug 2025 19:22:49 -0600 Subject: [PATCH 03/18] CI fixes --- .clang-tidy | 1 + .pre-commit-config.yaml | 2 +- include/pybind11/cast.h | 5 +- include/pybind11/detail/foreign.h | 99 +++++++---- include/pybind11/detail/internals.h | 5 +- include/pybind11/detail/pymetabind.h | 197 +++++++++++---------- include/pybind11/detail/type_caster_base.h | 42 +++-- include/pybind11/pybind11.h | 6 +- tests/extra_python_package/test_files.py | 2 + 9 files changed, 209 insertions(+), 150 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 3a1995c326..5285f80cad 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -78,3 +78,4 @@ CheckOptions: value: true HeaderFilterRegex: 'pybind11/.*h' +ExcludeHeaderFilterRegex: 'pybind11/detail/pymetabind.h' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8435fbac94..6d85d399f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ ci: autoupdate_schedule: monthly # third-party content -exclude: ^tools/JoinPaths.cmake$ +exclude: ^(tools/JoinPaths.cmake|include/pybind11/detail/pymetabind.h)$ repos: diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 28d95349c2..dadbaa5191 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -864,10 +864,11 @@ struct holder_helper { struct holder_caster_foreign_helpers { struct py_deleter { - void operator()(const void *) noexcept { + void operator()(const void *) const noexcept { // Don't run the deleter if the interpreter has been shut down - if (!Py_IsInitialized()) + if (Py_IsInitialized() == 0) { return; + } gil_scoped_acquire guard; Py_DECREF(o); } diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index c21357129e..6e5f6f6bc3 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -84,7 +84,7 @@ inline void *foreign_cb_from_python(pymb_binding *binding, type_caster_generic caster{static_cast(binding->context)}; void *ret = nullptr; try { - if (caster.load(pyobj, convert)) { + if (caster.load(pyobj, convert != 0)) { ret = caster.value; } } catch (...) { @@ -92,6 +92,7 @@ inline void *foreign_cb_from_python(pymb_binding *binding, PyErr_WriteUnraisable(pyobj); } if (keep_referenced) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) for (PyObject *item : holder->list_patients()) { keep_referenced(keep_referenced_ctx, item); } @@ -103,7 +104,7 @@ inline PyObject *foreign_cb_to_python(pymb_binding *binding, void *cobj, enum pymb_rv_policy rvp_, PyObject *parent) noexcept { - auto *ti = static_cast(binding->context); + const auto *ti = static_cast(binding->context); if (cobj == nullptr) { return none().release().ptr(); } @@ -177,8 +178,9 @@ inline void foreign_cb_translate_exception(const void *eptr) { // frameworks' exceptions), it's the second-last one and should // be skipped too. We don't want mutual recursion between // different frameworks' translators. - if (!get_foreign_internals().exc_frameworks.empty()) + if (!get_foreign_internals().exc_frameworks.empty()) { ++leader; + } for (; leader != exception_translators.end(); ++it, ++leader) { try { @@ -237,8 +239,9 @@ inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { should_remove_auto &= (it->second != binding->native_type); foreign_internals.manual_imports.erase(it); } - if (should_remove_auto) + if (should_remove_auto) { remove_from_type((const std::type_info *) binding->native_type); + } }); } @@ -255,15 +258,17 @@ inline void foreign_cb_add_foreign_framework(pymb_framework *framework) noexcept // translator). auto leader = exception_translators.begin(); auto trailer = exception_translators.before_begin(); - while (++leader != exception_translators.end()) + while (++leader != exception_translators.end()) { ++trailer; + } exception_translators.insert_after(trailer, foreign_exception_translator); } // Add the new framework at the end of the list auto leader = foreign_internals.exc_frameworks.begin(); auto trailer = leader; - while (++leader != foreign_internals.exc_frameworks.end()) + while (++leader != foreign_internals.exc_frameworks.end()) { ++trailer; + } foreign_internals.exc_frameworks.insert_after(trailer, framework); }); } @@ -274,13 +279,15 @@ inline void foreign_cb_add_foreign_framework(pymb_framework *framework) noexcept // Advertise our existence, and the above callbacks, to other frameworks PYBIND11_NOINLINE bool foreign_internals::initialize() { bool inited_by_us = with_internals([&](internals &) { - if (registry) + if (registry) { return false; + } registry = pymb_get_registry(); - if (!registry) + if (!registry) { throw error_already_set(); + } - self = std::make_unique(); + self.reset(new pymb_framework{}); self->name = "pybind11 " PYBIND11_ABI_TAG; // TODO: pybind11 does leak some bindings; there should be a way to // indicate that (so that eg nanobind can disable its leak detection) @@ -315,27 +322,32 @@ inline foreign_internals::~foreign_internals() = default; PYBIND11_NOINLINE void import_foreign_type(type pytype, const std::type_info *cpptype) { auto &foreign_internals = get_foreign_internals(); pymb_binding *binding = pymb_get_binding(pytype.ptr()); - if (!binding) + if (!binding) { pybind11_fail("pybind11::import_foreign_type(): type does not define " "a __pymetabind_binding__"); - if (binding->framework == foreign_internals.self.get()) + } + if (binding->framework == foreign_internals.self.get()) { pybind11_fail("pybind11::import_foreign_type(): type is not foreign"); + } if (!cpptype) { - if (binding->framework->abi_lang != pymb_abi_lang_cpp) + if (binding->framework->abi_lang != pymb_abi_lang_cpp) { pybind11_fail("pybind11::import_foreign_type(): type is not " "written in C++, so you must specify a C++ type"); - if (binding->framework->abi_extra != foreign_internals.self->abi_extra) + } + if (binding->framework->abi_extra != foreign_internals.self->abi_extra) { pybind11_fail("pybind11::import_foreign_type(): type has " "incompatible C++ ABI with this module"); + } cpptype = (const std::type_info *) binding->native_type; } auto result = foreign_internals.manual_imports.emplace(binding, cpptype); if (!result.second) { - auto *existing = (const std::type_info *) result.first->second; - if (existing != cpptype && *existing != *cpptype) + const auto *existing = (const std::type_info *) result.first->second; + if (existing != cpptype && *existing != *cpptype) { pybind11_fail("pybind11::import_foreign_type(): type was " "already imported as a different C++ type"); + } } import_foreign_binding(binding, cpptype); } @@ -346,13 +358,15 @@ PYBIND11_NOINLINE void import_foreign_type(type pytype, const std::type_info *cp PYBIND11_NOINLINE void foreign_enable_import_all() { auto &foreign_internals = get_foreign_internals(); bool proceed = with_internals([&](internals &) { - if (foreign_internals.import_all) + if (foreign_internals.import_all) { return false; + } foreign_internals.import_all = true; return true; }); - if (!proceed) + if (!proceed) { return; + } if (foreign_internals.initialize_if_needed()) { // pymb_add_framework tells us about every existing type when we // register, so if we register with import enabled, we're done @@ -361,11 +375,13 @@ PYBIND11_NOINLINE void foreign_enable_import_all() { // If we enable import after registering, we have to iterate over the // list of types ourselves. Do this without the internals lock held so // we can reuse the pymb callback functions. foreign_internals registry + - // self never change once they're non-null, so we can accesss them + // self never change once they're non-null, so we can access them // without locking here. pymb_lock_registry(foreign_internals.registry); + // NOLINTNEXTLINE(modernize-use-auto) PYMB_LIST_FOREACH(struct pymb_binding *, binding, foreign_internals.registry->bindings) { - if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) { + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding) != 0) { foreign_cb_add_foreign_binding(binding); pymb_unref_binding(binding); } @@ -379,9 +395,11 @@ PYBIND11_NOINLINE void foreign_enable_import_all() { PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { auto &foreign_internals = get_foreign_internals(); auto range = foreign_internals.bindings.equal_range(*ti->cpptype); - for (auto it = range.first; it != range.second; ++it) - if (it->second->framework == foreign_internals.self.get()) + for (auto it = range.first; it != range.second; ++it) { + if (it->second->framework == foreign_internals.self.get()) { return; // already exported + } + } auto *binding = new pymb_binding{}; binding->framework = foreign_internals.self.get(); @@ -391,7 +409,7 @@ PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { binding->context = ti; capsule tie_lifetimes((void *) binding, [](void *p) { - pymb_binding *binding = (pymb_binding *) p; + auto *binding = (pymb_binding *) p; pymb_remove_binding(get_foreign_internals().registry, binding); free(const_cast(binding->source_name)); delete binding; @@ -407,14 +425,16 @@ PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { PYBIND11_NOINLINE void foreign_enable_export_all() { auto &foreign_internals = get_foreign_internals(); bool proceed = with_internals([&](internals &) { - if (foreign_internals.export_all) + if (foreign_internals.export_all) { return false; + } foreign_internals.export_all = true; foreign_internals.export_type_to_foreign = &detail::export_type_to_foreign; return true; }); - if (!proceed) + if (!proceed) { return; + } foreign_internals.initialize_if_needed(); with_internals([&](internals &internals) { for (const auto &entry : internals.registered_types_cpp) { @@ -434,15 +454,18 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, auto &foreign_internals = get_foreign_internals(); PYBIND11_LOCK_INTERNALS(internals); + (void) internals; // suppress unused warning on non-ft builds auto range = foreign_internals.bindings.equal_range(*type); - if (range.first == range.second) + if (range.first == range.second) { return nullptr; // no foreign bindings + } if (std::next(range.first) == range.second) { // Single binding - check that it's not our own auto *binding = range.first->second; - if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) { + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding) != 0) { #ifdef Py_GIL_DISABLED // attempt() might execute Python code; drop the internals lock // to avoid a deadlock @@ -459,11 +482,13 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, #ifndef Py_GIL_DISABLED for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) { + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding) != 0) { void *result = attempt(closure, binding); pymb_unref_binding(binding); - if (result) + if (result) { return result; + } } } return nullptr; @@ -486,8 +511,10 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, // our scratch storage for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() && pymb_try_ref_binding(binding)) + if (binding->framework != foreign_internals.self.get() && + pymb_try_ref_binding(binding) != 0) { *scratch_tail++ = binding; + } } // Drop the lock and proceed using only our saved binding pointers. @@ -496,8 +523,9 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, lock.unlock(); void *result = nullptr; while (scratch != scratch_tail) { - if (!result) + if (!result) { result = attempt(closure, *scratch); + } pymb_unref_binding(*scratch); ++scratch; } @@ -509,10 +537,12 @@ PYBIND11_NAMESPACE_END(detail) inline void set_foreign_type_defaults(bool export_all, bool import_all) { auto &foreign_internals = detail::get_foreign_internals(); - if (import_all && !foreign_internals.import_all) + if (import_all && !foreign_internals.import_all) { detail::foreign_enable_import_all(); - if (export_all && !foreign_internals.export_all) + } + if (export_all && !foreign_internals.export_all) { detail::foreign_enable_export_all(); + } } template @@ -521,14 +551,15 @@ inline void import_foreign_type(type pytype) { auto &foreign_internals = detail::get_foreign_internals(); foreign_internals.initialize_if_needed(); detail::with_internals( - [&](detail::internals &) { detail::import_foreign_type(pytype, cpptype); }); + [&](detail::internals &) { detail::import_foreign_type(std::move(pytype), cpptype); }); } inline void export_type_to_foreign(type ty) { detail::type_info *ti = detail::get_type_info((PyTypeObject *) ty.ptr()); - if (!ti) + if (!ti) { pybind11_fail("pybind11::export_type_to_foreign: not a " "pybind11 registered type"); + } auto &foreign_internals = detail::get_foreign_internals(); foreign_internals.initialize_if_needed(); detail::with_internals([&](detail::internals &) { detail::export_type_to_foreign(ti); }); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 6d5bdde885..0aa24a38e4 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -363,13 +363,14 @@ struct foreign_internals { // Returns true if we initialized, false if someone else already did. inline bool initialize_if_needed() { - if (registry) + if (registry) { return false; + } return initialize(); } private: - inline bool initialize(); + bool initialize(); }; // the internals struct (above) is shared between all the modules. local_internals are only diff --git a/include/pybind11/detail/pymetabind.h b/include/pybind11/detail/pymetabind.h index e026f01012..d42cd9c7eb 100644 --- a/include/pybind11/detail/pymetabind.h +++ b/include/pybind11/detail/pymetabind.h @@ -38,11 +38,11 @@ #pragma once -#include #include +#include #if !defined(PY_VERSION_HEX) -# error You must include Python.h before this header +# error You must include Python.h before this header #endif /* @@ -59,7 +59,7 @@ * compilation unit that doesn't request `PYMB_DECLS_ONLY`. */ #if !defined(PYMB_FUNC) -# define PYMB_FUNC inline +#define PYMB_FUNC inline #endif #if defined(__cplusplus) @@ -132,11 +132,11 @@ struct pymb_list { struct pymb_list_node head; }; -inline void pymb_list_init(struct pymb_list *list) { +inline void pymb_list_init(struct pymb_list* list) { list->head.prev = list->head.next = &list->head; } -inline void pymb_list_unlink(struct pymb_list_node *node) { +inline void pymb_list_unlink(struct pymb_list_node* node) { if (node->next) { node->next->prev = node->prev; node->prev->next = node->next; @@ -144,17 +144,19 @@ inline void pymb_list_unlink(struct pymb_list_node *node) { } } -inline void pymb_list_append(struct pymb_list *list, struct pymb_list_node *node) { +inline void pymb_list_append(struct pymb_list* list, + struct pymb_list_node* node) { pymb_list_unlink(node); - struct pymb_list_node *tail = list->head.prev; + struct pymb_list_node* tail = list->head.prev; tail->next = node; list->head.prev = node; node->prev = tail; node->next = &list->head; } -#define PYMB_LIST_FOREACH(type, name, list) \ - for (type name = (type) (list).head.next; name != (type) & (list).head; \ +#define PYMB_LIST_FOREACH(type, name, list) \ + for (type name = (type) (list).head.next; \ + name != (type) &(list).head; \ name = (type) name->hook.next) /* @@ -194,13 +196,15 @@ struct pymb_registry { }; #if defined(Py_GIL_DISALED) -inline void pymb_lock_registry(struct pymb_registry *registry) { PyMutex_Lock(®istry->mutex); } -inline void pymb_unlock_registry(struct pymb_registry *registry) { +inline void pymb_lock_registry(struct pymb_registry* registry) { + PyMutex_Lock(®istry->mutex); +} +inline void pymb_unlock_registry(struct pymb_registry* registry) { PyMutex_Unlock(®istry->mutex); } #else -inline void pymb_lock_registry(struct pymb_registry *) {} -inline void pymb_unlock_registry(struct pymb_registry *) {} +inline void pymb_lock_registry(struct pymb_registry*) {} +inline void pymb_unlock_registry(struct pymb_registry*) {} #endif struct pymb_binding; @@ -237,7 +241,7 @@ struct pymb_framework { struct pymb_list_node hook; // Human-readable description of this framework, as a NUL-terminated string - const char *name; + const char* name; // Does this framework guarantee that its `pymb_binding` structures remain // valid to use for the lifetime of the Python interpreter process once @@ -273,7 +277,7 @@ struct pymb_framework { // form of interning to speed up checking that a given binding is usable. // Thus, to check whether another framework's ABI matches yours, you can // do a pointer comparison `me->abi_extra == them->abi_extra`. - const char *abi_extra; + const char* abi_extra; // The function pointers below allow other frameworks to interact with // bindings provided by this framework. They are constant after construction @@ -303,11 +307,11 @@ struct pymb_framework { // On free-threaded builds, callers must ensure that the `binding` is not // destroyed during a call to `from_python`. The requirements for this are // subtle; see the full discussion in the comment for `struct pymb_binding`. - void *(*from_python)(struct pymb_binding *binding, - PyObject *pyobj, + void* (*from_python)(struct pymb_binding* binding, + PyObject* pyobj, uint8_t convert, - void (*keep_referenced)(void *ctx, PyObject *obj), - void *keep_referenced_ctx); + void (*keep_referenced)(void* ctx, PyObject* obj), + void* keep_referenced_ctx); // Wrap the C/C++/etc object `cobj` into a Python object using the given // return value policy. The type is specified by providing a `pymb_binding*` @@ -324,10 +328,10 @@ struct pymb_framework { // On free-threaded builds, callers must ensure that the `binding` is not // destroyed during a call to `to_python`. The requirements for this are // subtle; see the full discussion in the comment for `struct pymb_binding`. - PyObject *(*to_python)(struct pymb_binding *binding, - void *cobj, + PyObject* (*to_python)(struct pymb_binding* binding, + void* cobj, enum pymb_rv_policy rvp, - PyObject *parent); + PyObject* parent); // Request that a PyObject reference be dropped, or that a callback // be invoked, when `nurse` is destroyed. `nurse` should be an object @@ -337,7 +341,7 @@ struct pymb_framework { // or -1 and sets the Python error indicator on error. // // No synchronization is required to call this method. - int (*keep_alive)(PyObject *nurse, void *payload, void (*cb)(void *)); + int (*keep_alive)(PyObject* nurse, void* payload, void (*cb)(void*)); // Attempt to translate a C++ exception known to this framework to Python. // This should translate only framework-specific exceptions or user-defined @@ -349,28 +353,28 @@ struct pymb_framework { // C++ exception translation. // // No synchronization is required to call this method. - void (*translate_exception)(const void *eptr); + void (*translate_exception)(const void* eptr); // Notify this framework that some other framework published a new binding. // This call will be made after the new binding has been linked into the // `pymb_registry::bindings` list. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*add_foreign_binding)(struct pymb_binding *binding); + void (*add_foreign_binding)(struct pymb_binding* binding); // Notify this framework that some other framework is about to remove // a binding. This call will be made after the binding has been removed // from the `pymb_registry::bindings` list. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*remove_foreign_binding)(struct pymb_binding *binding); + void (*remove_foreign_binding)(struct pymb_binding* binding); // Notify this framework that some other framework came into existence. // This call will be made after the new framework has been linked into the // `pymb_registry::frameworks` list and before it adds any bindings. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*add_foreign_framework)(struct pymb_framework *framework); + void (*add_foreign_framework)(struct pymb_framework* framework); // There is no remove_foreign_framework(); the interpreter has // already been finalized at that point, so there's nothing for the @@ -473,26 +477,26 @@ struct pymb_binding { struct pymb_list_node hook; // The framework that provides this binding - struct pymb_framework *framework; + struct pymb_framework* framework; // Python type: you will get an instance of this type from a successful // call to `framework::from_python()` that passes this binding - PyTypeObject *pytype; + PyTypeObject* pytype; // The native identifier for this type in `framework->abi_lang`, if that is // a concept that exists in that language. See the documentation of // `enum pymb_abi_lang` for specific per-language semantics. - const void *native_type; + const void* native_type; // The way that this type would be written in `framework->abi_lang` source // code, as a NUL-terminated byte string without struct/class/enum words. // Examples: "Foo", "Bar::Baz", "std::vector >" - const char *source_name; + const char* source_name; // Pointer that is free for use by the framework, e.g., to point to its // own data about this type. If the framework needs more data, it can // over-allocate the `pymb_binding` storage and use the space after this. - void *context; + void* context; }; /* @@ -501,16 +505,18 @@ struct pymb_binding { * considered part of the ABI. */ -PYMB_FUNC struct pymb_registry *pymb_get_registry(); -PYMB_FUNC void pymb_add_framework(struct pymb_registry *registry, - struct pymb_framework *framework); -PYMB_FUNC void pymb_remove_framework(struct pymb_registry *registry, - struct pymb_framework *framework); -PYMB_FUNC void pymb_add_binding(struct pymb_registry *registry, struct pymb_binding *binding); -PYMB_FUNC void pymb_remove_binding(struct pymb_registry *registry, struct pymb_binding *binding); -PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding *binding); -PYMB_FUNC void pymb_unref_binding(struct pymb_binding *binding); -PYMB_FUNC struct pymb_binding *pymb_get_binding(PyObject *type); +PYMB_FUNC struct pymb_registry* pymb_get_registry(); +PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, + struct pymb_framework* framework); +PYMB_FUNC void pymb_remove_framework(struct pymb_registry* registry, + struct pymb_framework* framework); +PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, + struct pymb_binding* binding); +PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, + struct pymb_binding* binding); +PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding); +PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding); +PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); #if !defined(PYMB_DECLS_ONLY) @@ -520,26 +526,27 @@ PYMB_FUNC struct pymb_binding *pymb_get_binding(PyObject *type); * This must be called from a module initialization function so that the * import lock can provide mutual exclusion. */ -PYMB_FUNC struct pymb_registry *pymb_get_registry() { -# if defined(PYPY_VERSION) - PyObject *dict = PyEval_GetBuiltins(); -# elif PY_VERSION_HEX < 0x03090000 - PyObject *dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); -# else - PyObject *dict = PyInterpreterState_GetDict(PyInterpreterState_Get()); -# endif - PyObject *key = PyUnicode_FromString("__pymetabind_registry__"); +PYMB_FUNC struct pymb_registry* pymb_get_registry() { +#if defined(PYPY_VERSION) + PyObject* dict = PyEval_GetBuiltins(); +#elif PY_VERSION_HEX < 0x03090000 + PyObject* dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); +#else + PyObject* dict = PyInterpreterState_GetDict(PyInterpreterState_Get()); +#endif + PyObject* key = PyUnicode_FromString("__pymetabind_registry__"); if (!dict || !key) { Py_XDECREF(key); return NULL; } - PyObject *capsule = PyDict_GetItem(dict, key); + PyObject* capsule = PyDict_GetItem(dict, key); if (capsule) { Py_DECREF(key); - return (struct pymb_registry *) PyCapsule_GetPointer(capsule, "pymetabind_registry"); + return (struct pymb_registry*) PyCapsule_GetPointer( + capsule, "pymetabind_registry"); } - struct pymb_registry *registry; - registry = (struct pymb_registry *) calloc(1, sizeof(*registry)); + struct pymb_registry* registry; + registry = (struct pymb_registry*) calloc(1, sizeof(*registry)); if (registry) { pymb_list_init(®istry->frameworks); pymb_list_init(®istry->bindings); @@ -562,30 +569,30 @@ PYMB_FUNC struct pymb_registry *pymb_get_registry() { * framework->add_foreign_framework() and framework->add_foreign_binding() * for each existing framework/binding in the registry. */ -PYMB_FUNC void pymb_add_framework(struct pymb_registry *registry, - struct pymb_framework *framework) { -# if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030e0000 - assert(framework->bindings_usable_forever - && "Free-threaded removal of bindings requires PyUnstable_TryIncRef(), " - "which was added in CPython 3.14"); -# endif +PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, + struct pymb_framework* framework) { +#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030e0000 + assert(framework->bindings_usable_forever && + "Free-threaded removal of bindings requires PyUnstable_TryIncRef(), " + "which was added in CPython 3.14"); +#endif pymb_lock_registry(registry); - PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { // Intern `abi_extra` strings so they can be compared by pointer - if (other->abi_extra && framework->abi_extra - && 0 == strcmp(other->abi_extra, framework->abi_extra)) { + if (other->abi_extra && framework->abi_extra && + 0 == strcmp(other->abi_extra, framework->abi_extra)) { framework->abi_extra = other->abi_extra; break; } } pymb_list_append(®istry->frameworks, &framework->hook); - PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { if (other != framework) { other->add_foreign_framework(framework); framework->add_foreign_framework(other); } } - PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { + PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { if (binding->framework != framework && pymb_try_ref_binding(binding)) { framework->add_foreign_binding(binding); pymb_unref_binding(binding); @@ -595,15 +602,16 @@ PYMB_FUNC void pymb_add_framework(struct pymb_registry *registry, } /* Add a new binding to the given registry */ -PYMB_FUNC void pymb_add_binding(struct pymb_registry *registry, struct pymb_binding *binding) { -# if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000 +PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, + struct pymb_binding* binding) { +#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000 PyUnstable_EnableTryIncRef((PyObject *) binding->pytype); -# endif - PyObject *capsule = PyCapsule_New(binding, "pymetabind_binding", NULL); +#endif + PyObject* capsule = PyCapsule_New(binding, "pymetabind_binding", NULL); int rv = -1; if (capsule) { - rv = PyObject_SetAttrString( - (PyObject *) binding->pytype, "__pymetabind_binding__", capsule); + rv = PyObject_SetAttrString((PyObject *) binding->pytype, + "__pymetabind_binding__", capsule); Py_DECREF(capsule); } if (rv != 0) { @@ -611,7 +619,7 @@ PYMB_FUNC void pymb_add_binding(struct pymb_registry *registry, struct pymb_bind } pymb_lock_registry(registry); pymb_list_append(®istry->bindings, &binding->hook); - PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { if (other != binding->framework) { other->add_foreign_binding(binding); } @@ -625,10 +633,11 @@ PYMB_FUNC void pymb_add_binding(struct pymb_registry *registry, struct pymb_bind * zero but still accessible. Once this function returns, you can free the * binding structure. */ -PYMB_FUNC void pymb_remove_binding(struct pymb_registry *registry, struct pymb_binding *binding) { +PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, + struct pymb_binding* binding) { pymb_lock_registry(registry); pymb_list_unlink(&binding->hook); - PYMB_LIST_FOREACH(struct pymb_framework *, other, registry->frameworks) { + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { if (other != binding->framework) { other->remove_foreign_binding(binding); } @@ -641,56 +650,56 @@ PYMB_FUNC void pymb_remove_binding(struct pymb_registry *registry, struct pymb_b * use the binding and must call pymb_unref_binding() when done) or 0 if the * binding is being removed and shouldn't be used. */ -PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding *binding) { -# if defined(Py_GIL_DISABLED) +PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding) { +#if defined(Py_GIL_DISABLED) if (!binding->framework->bindings_usable_forever) { -# if PY_VERSION_HEX >= 0x030e0000 +#if PY_VERSION_HEX >= 0x030e0000 return PyUnstable_TryIncRef((PyObject *) binding->pytype); -# else +#else // bindings_usable_forever is required on this Python version, and // was checked in pymb_add_framework() assert(false); -# endif +#endif } -# else +#else Py_INCREF((PyObject *) binding->pytype); -# endif +#endif return 1; } /* Decrease the reference count of a binding. */ -PYMB_FUNC void pymb_unref_binding(struct pymb_binding *binding) { -# if defined(Py_GIL_DISABLED) +PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding) { +#if defined(Py_GIL_DISABLED) if (!binding->framework->bindings_usable_forever) { -# if PY_VERSION_HEX >= 0x030e0000 +#if PY_VERSION_HEX >= 0x030e0000 Py_DECREF((PyObject *) binding->pytype); -# else +#else // bindings_usable_forever is required on this Python version, and // was checked in pymb_add_framework() assert(false); -# endif +#endif } -# else +#else Py_DECREF((PyObject *) binding->pytype); -# endif +#endif } /* * Return a pointer to a pymb_binding for the Python type `type`, or NULL if * none exists. */ -PYMB_FUNC struct pymb_binding *pymb_get_binding(PyObject *type) { - PyObject *capsule = PyObject_GetAttrString(type, "__pymetabind_binding__"); +PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type) { + PyObject* capsule = PyObject_GetAttrString(type, "__pymetabind_binding__"); if (capsule == NULL) { PyErr_Clear(); return NULL; } - void *binding = PyCapsule_GetPointer(capsule, "pymetabind_binding"); + void* binding = PyCapsule_GetPointer(capsule, "pymetabind_binding"); Py_DECREF(capsule); if (!binding) { PyErr_Clear(); } - return (struct pymb_binding *) binding; + return (struct pymb_binding*) binding; } #endif /* defined(PYMB_DECLS_ONLY) */ diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 773cf8ce14..f4f3085da4 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -253,16 +253,19 @@ PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, if (foreign_internals.imported_any) { handle ret = with_internals([&](internals &) { auto range = foreign_internals.bindings.equal_range(tp); - if (range.first != range.second) + if (range.first != range.second) { return handle((PyObject *) range.first->second->pytype); + } return handle(); }); - if (ret) + if (ret) { return ret; + } } } - if (throw_if_missing) + if (throw_if_missing) { return handle((PyObject *) get_type_info(tp, true)->type); + } return nullptr; } @@ -911,9 +914,10 @@ class type_caster_generic { // Use the given pointer with its compile-time type, possibly downcast // via polymorphic_type_hook() template - cast_sources(const itype *ptr); + cast_sources(const itype *ptr); // NOLINT(google-explicit-constructor) // Use the given pointer and type + // NOLINTNEXTLINE(google-explicit-constructor) cast_sources(const source &orig) : original(orig) { result = resolve(); } // Use the given object and pybind11 type info. NB: if tinfo is null, @@ -954,13 +958,15 @@ class type_caster_generic { private: PYBIND11_NOINLINE std::pair resolve() { if (downcast.type) { - if (same_type(*original.type, *downcast.type)) + if (same_type(*original.type, *downcast.type)) { const_cast(downcast.type) = nullptr; - else if (const auto *tpi = get_type_info(*downcast.type)) + } else if (const auto *tpi = get_type_info(*downcast.type)) { return {downcast.obj, tpi}; + } } - if (const auto *tpi = get_type_info(*original.type)) + if (const auto *tpi = get_type_info(*original.type)) { return {original.obj, tpi}; + } return {nullptr, nullptr}; } }; @@ -989,8 +995,9 @@ class type_caster_generic { capture &cap = *(capture *) closure; void *ret = binding->framework->to_python( binding, const_cast(cap.src), cap.policy, cap.parent); - if (ret) + if (ret) { *cap.used_foreign = binding->framework; + } return ret; }; @@ -1009,12 +1016,14 @@ class type_caster_generic { } if (srcs.downcast.type) { cap.src = srcs.downcast.obj; - if (void *result = try_foreign_bindings(srcs.downcast.type, attempt, &cap)) + if (void *result = try_foreign_bindings(srcs.downcast.type, attempt, &cap)) { return (PyObject *) result; + } } cap.src = srcs.original.obj; - if (void *result = try_foreign_bindings(srcs.original.type, attempt, &cap)) + if (void *result = try_foreign_bindings(srcs.original.type, attempt, &cap)) { return (PyObject *) result; + } return nullptr; } @@ -1028,8 +1037,9 @@ class type_caster_generic { // No pybind11 type info. See if we can use another framework's // type to complete this cast. Set srcs.used_foreign if so. if (get_foreign_internals().imported_any) { - if (handle ret = cast_foreign(srcs, policy, parent)) + if (handle ret = cast_foreign(srcs, policy, parent)) { return ret; + } } std::string tname = srcs.downcast.type ? srcs.downcast.type->name() : srcs.original.type ? srcs.original.type->name() @@ -1192,8 +1202,9 @@ class type_caster_generic { /// Try to load as a type exposed by a different binding framework. bool try_load_other_framework(handle src, bool convert) { auto &foreign_internals = get_foreign_internals(); - if (!foreign_internals.imported_any || !cpptype || src.is_none()) + if (!foreign_internals.imported_any || !cpptype || src.is_none()) { return false; + } struct capture { handle src; @@ -1667,6 +1678,7 @@ class type_caster_base : public type_caster_generic { explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) {} struct cast_sources : type_caster_generic::cast_sources { + // NOLINTNEXTLINE(google-explicit-constructor) cast_sources(const itype *ptr) : type_caster_generic::cast_sources(ptr) {} }; @@ -1696,7 +1708,7 @@ class type_caster_base : public type_caster_generic { srcs, return_value_policy::take_ownership, {}, nullptr, nullptr, holder); if (srcs.used_foreign) { // Foreign cast succeeded; release C++ ownership - holder->release(); + (void) holder->release(); } return ret; } @@ -1705,14 +1717,14 @@ class type_caster_base : public type_caster_generic { handle ret, std::shared_ptr holder, pymb_framework *framework) { // Make the resulting Python object keep a shared_ptr alive, // even if there's not space for it inside the object. - auto sp = std::make_unique>(std::move(holder)); + std::unique_ptr> sp{new auto{std::move(holder)}}; if (-1 == framework->keep_alive(ret.ptr(), sp.get(), [](void *p) noexcept { delete (std::shared_ptr *) p; })) { ret.dec_ref(); throw error_already_set(); } - sp.release(); + (void) sp.release(); } template diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 548e5e1e38..53e8cfd8cc 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2940,9 +2940,11 @@ PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { if (Py_TYPE(nurse.ptr())->tp_weaklistoffset == 0) { // The nurse type is not weak-referenceable. Maybe it is a // different framework's type; try to get them to do the keep_alive. - if (auto *binding = pymb_get_binding(type::handle_of(nurse).ptr())) - if (0 != binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr)) + if (auto *binding = pymb_get_binding(type::handle_of(nurse).ptr())) { + if (0 != binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr)) { throw error_already_set(); + } + } // Otherwise continue with the logic below (which will // raise an error). } diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 63e59f65a6..90e9fc5d6b 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -81,11 +81,13 @@ "include/pybind11/detail/cpp_conduit.h", "include/pybind11/detail/descr.h", "include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h", + "include/pybind11/detail/foreign.h", "include/pybind11/detail/function_record_pyobject.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", "include/pybind11/detail/native_enum_data.h", "include/pybind11/detail/pybind11_namespace_macros.h", + "include/pybind11/detail/pymetabind.h", "include/pybind11/detail/struct_smart_holder.h", "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/typeid.h", From eab23fa5ab598365daa322c634125a8d61e43e38 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 01:24:13 +0000 Subject: [PATCH 04/18] style: pre-commit fixes --- include/pybind11/detail/foreign.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index 6e5f6f6bc3..5e9ec650a4 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -380,8 +380,8 @@ PYBIND11_NOINLINE void foreign_enable_import_all() { pymb_lock_registry(foreign_internals.registry); // NOLINTNEXTLINE(modernize-use-auto) PYMB_LIST_FOREACH(struct pymb_binding *, binding, foreign_internals.registry->bindings) { - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding) != 0) { + if (binding->framework != foreign_internals.self.get() + && pymb_try_ref_binding(binding) != 0) { foreign_cb_add_foreign_binding(binding); pymb_unref_binding(binding); } @@ -464,8 +464,8 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, if (std::next(range.first) == range.second) { // Single binding - check that it's not our own auto *binding = range.first->second; - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding) != 0) { + if (binding->framework != foreign_internals.self.get() + && pymb_try_ref_binding(binding) != 0) { #ifdef Py_GIL_DISABLED // attempt() might execute Python code; drop the internals lock // to avoid a deadlock @@ -482,8 +482,8 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, #ifndef Py_GIL_DISABLED for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding) != 0) { + if (binding->framework != foreign_internals.self.get() + && pymb_try_ref_binding(binding) != 0) { void *result = attempt(closure, binding); pymb_unref_binding(binding); if (result) { @@ -511,8 +511,8 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, // our scratch storage for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() && - pymb_try_ref_binding(binding) != 0) { + if (binding->framework != foreign_internals.self.get() + && pymb_try_ref_binding(binding) != 0) { *scratch_tail++ = binding; } } From aab695985eaae553a7e5a6116a8f23e7541d41f5 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sun, 17 Aug 2025 19:46:53 -0600 Subject: [PATCH 05/18] CI fixes --- include/pybind11/detail/common.h | 6 ++++++ include/pybind11/detail/foreign.h | 2 +- include/pybind11/detail/type_caster_base.h | 2 +- include/pybind11/pybind11.h | 8 +------- tests/test_buffers.cpp | 2 +- tests/test_class.cpp | 1 + tests/test_class_release_gil_before_calling_cpp_dtor.cpp | 1 + tests/test_cross_module_rtti/lib.h | 1 + tests/test_eigen_matrix.cpp | 1 + tests/test_numpy_array.cpp | 1 + tests/test_potentially_slicing_weak_ptr.cpp | 1 + tests/test_smart_ptr.cpp | 6 ++++++ tests/test_stl.cpp | 1 + tests/test_tagbased_polymorphic.cpp | 2 ++ 14 files changed, 25 insertions(+), 10 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index ed47a3f46d..93ad0444a3 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -214,6 +214,12 @@ # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif +#if defined(_MSC_VER) +# define PYBIND11_COMPAT_STRDUP _strdup +#else +# define PYBIND11_COMPAT_STRDUP strdup +#endif + #include #include #include diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index 5e9ec650a4..2a52292f1e 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -405,7 +405,7 @@ PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { binding->framework = foreign_internals.self.get(); binding->pytype = ti->type; binding->native_type = ti->cpptype; - binding->source_name = strdup(clean_type_id(ti->cpptype->name()).c_str()); + binding->source_name = PYBIND11_COMPAT_STRDUP(clean_type_id(ti->cpptype->name()).c_str()); binding->context = ti; capsule tie_lifetimes((void *) binding, [](void *p) { diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index f4f3085da4..93a7e407c5 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -1717,7 +1717,7 @@ class type_caster_base : public type_caster_generic { handle ret, std::shared_ptr holder, pymb_framework *framework) { // Make the resulting Python object keep a shared_ptr alive, // even if there's not space for it inside the object. - std::unique_ptr> sp{new auto{std::move(holder)}}; + std::unique_ptr> sp{new auto(std::move(holder))}; if (-1 == framework->keep_alive(ret.ptr(), sp.get(), [](void *p) noexcept { delete (std::shared_ptr *) p; })) { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 53e8cfd8cc..ae10656bc4 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -161,7 +161,7 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel if (handle th = detail::get_type_handle(*t, false, true)) { signature += th.attr("__module__").cast() + "." + th.attr("__qualname__").cast(); - } else if (auto th = detail::global_internals_native_enum_type_map_get_item(*t)) { + } else if ((th = detail::global_internals_native_enum_type_map_get_item(*t))) { signature += th.attr("__module__").cast() + "." + th.attr("__qualname__").cast(); } else if (func_rec->is_new_style_constructor && arg_index == 0) { @@ -242,12 +242,6 @@ inline std::string generate_type_signature() { caster_name_field.text, &func_rec, descr_types.data(), type_index, arg_index); } -#if defined(_MSC_VER) -# define PYBIND11_COMPAT_STRDUP _strdup -#else -# define PYBIND11_COMPAT_STRDUP strdup -#endif - PYBIND11_NAMESPACE_END(detail) /// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index a090c8745f..8c9f9c2b94 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -227,7 +227,7 @@ TEST_SUBMODULE(buffers, m) { + std::to_string(cols) + "(*" + std::to_string(col_factor) + ") matrix"); } - + DiscontiguousMatrix(const DiscontiguousMatrix&) = delete; ~DiscontiguousMatrix() { print_destroyed(this, std::to_string(rows() / m_row_factor) + "(*" diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 3d567fc1f5..40c57df2ac 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -521,6 +521,7 @@ TEST_SUBMODULE(class_, m) { // test_exception_rvalue_abort struct PyPrintDestructor { PyPrintDestructor() = default; + PyPrintDestructor(const PyPrintDestructor&) = default; ~PyPrintDestructor() { py::print("Print from destructor"); } void throw_something() { throw std::runtime_error("error"); } }; diff --git a/tests/test_class_release_gil_before_calling_cpp_dtor.cpp b/tests/test_class_release_gil_before_calling_cpp_dtor.cpp index a4869a846f..713b5015c2 100644 --- a/tests/test_class_release_gil_before_calling_cpp_dtor.cpp +++ b/tests/test_class_release_gil_before_calling_cpp_dtor.cpp @@ -22,6 +22,7 @@ struct ProbeType { public: explicit ProbeType(const std::string &unique_key) : unique_key{unique_key} {} + ProbeType(const ProbeType&) = default; ~ProbeType() { RegistryType ® = PyGILState_Check_Results(); diff --git a/tests/test_cross_module_rtti/lib.h b/tests/test_cross_module_rtti/lib.h index 0925b084ca..e11dd19dcf 100644 --- a/tests/test_cross_module_rtti/lib.h +++ b/tests/test_cross_module_rtti/lib.h @@ -12,6 +12,7 @@ __pragma(warning(disable : 4251)) class TEST_CROSS_MODULE_RTTI_LIB_EXPORT Base : public std::enable_shared_from_this { public: Base(int a, int b); + Base(const Base&) = default; virtual ~Base() = default; virtual int get() const; diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index 4e6689a797..d4e83bdc47 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -237,6 +237,7 @@ TEST_SUBMODULE(eigen_matrix, m) { public: ReturnTester() { print_created(this); } + ReturnTester(const ReturnTester&) = default; ~ReturnTester() { print_destroyed(this); } static Eigen::MatrixXd create() { return Eigen::MatrixXd::Ones(10, 10); } // NOLINTNEXTLINE(readability-const-return-type) diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 1bfca33bb6..3f75f75101 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -282,6 +282,7 @@ TEST_SUBMODULE(numpy_array, sm) { struct ArrayClass { int data[2] = {1, 2}; ArrayClass() { py::print("ArrayClass()"); } + ArrayClass(const ArrayClass&) = default; ~ArrayClass() { py::print("~ArrayClass()"); } }; py::class_(sm, "ArrayClass") diff --git a/tests/test_potentially_slicing_weak_ptr.cpp b/tests/test_potentially_slicing_weak_ptr.cpp index 01b147faff..57de2fa102 100644 --- a/tests/test_potentially_slicing_weak_ptr.cpp +++ b/tests/test_potentially_slicing_weak_ptr.cpp @@ -12,6 +12,7 @@ namespace potentially_slicing_weak_ptr { template // Using int as a trick to easily generate multiple types. struct VirtBase { virtual ~VirtBase() = default; + VirtBase(const VirtBase&) = delete; virtual int get_code() { return 100; } }; diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp index 5fdd69db38..2db5293637 100644 --- a/tests/test_smart_ptr.cpp +++ b/tests/test_smart_ptr.cpp @@ -161,6 +161,8 @@ class MyObject4a { print_created(this); pointer_set().insert(this); }; + MyObject4a(const MyObject4a&) = delete; + int value; static void cleanupAllInstances() { @@ -182,6 +184,7 @@ class MyObject4a { class MyObject4b : public MyObject4a { public: explicit MyObject4b(int i) : MyObject4a(i) { print_created(this); } + MyObject4b(const MyObject4b&) = delete; ~MyObject4b() override { print_destroyed(this); } }; @@ -189,6 +192,7 @@ class MyObject4b : public MyObject4a { class MyObject5 { // managed by huge_unique_ptr public: explicit MyObject5(int value) : value{value} { print_created(this); } + MyObject5(const MyObject5&) = delete; ~MyObject5() { print_destroyed(this); } int value; }; @@ -245,6 +249,7 @@ struct SharedFromThisVirt : virtual SharedFromThisVBase {}; // test_move_only_holder struct C { C() { print_created(this); } + C(const C&) = delete; ~C() { print_destroyed(this); } }; @@ -265,6 +270,7 @@ struct TypeForHolderWithAddressOf { // test_move_only_holder_with_addressof_operator struct TypeForMoveOnlyHolderWithAddressOf { explicit TypeForMoveOnlyHolderWithAddressOf(int value) : value{value} { print_created(this); } + TypeForMoveOnlyHolderWithAddressOf(const TypeForMoveOnlyHolderWithAddressOf &) = delete; ~TypeForMoveOnlyHolderWithAddressOf() { print_destroyed(this); } std::string toString() const { return "MoveOnlyHolderWithAddressOf[" + std::to_string(value) + "]"; diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 5e6d6a333f..969af98dec 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -99,6 +99,7 @@ class OptionalProperties { using OptionalEnumValue = OptionalImpl; OptionalProperties() : value(EnumType::kSet) {} + OptionalProperties(const OptionalProperties&) = default; ~OptionalProperties() { // Reset value to detect use-after-destruction. // This is set to a specific value rather than nullopt to ensure that diff --git a/tests/test_tagbased_polymorphic.cpp b/tests/test_tagbased_polymorphic.cpp index 13e5ed3198..1d709732f0 100644 --- a/tests/test_tagbased_polymorphic.cpp +++ b/tests/test_tagbased_polymorphic.cpp @@ -17,6 +17,8 @@ struct Animal { // (https://github.com/pybind/pybind11/pull/2016/). virtual ~Animal() = default; + Animal(const Animal&) = delete; + // Enum for tag-based polymorphism. enum class Kind { Unknown = 0, From aafb8787c28c235c0f7fcf976187dacc95fc5f19 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 01:47:28 +0000 Subject: [PATCH 06/18] style: pre-commit fixes --- tests/test_buffers.cpp | 2 +- tests/test_class.cpp | 2 +- tests/test_class_release_gil_before_calling_cpp_dtor.cpp | 2 +- tests/test_cross_module_rtti/lib.h | 2 +- tests/test_eigen_matrix.cpp | 2 +- tests/test_numpy_array.cpp | 2 +- tests/test_potentially_slicing_weak_ptr.cpp | 2 +- tests/test_smart_ptr.cpp | 8 ++++---- tests/test_stl.cpp | 2 +- tests/test_tagbased_polymorphic.cpp | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 8c9f9c2b94..b11d1bb31f 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -227,7 +227,7 @@ TEST_SUBMODULE(buffers, m) { + std::to_string(cols) + "(*" + std::to_string(col_factor) + ") matrix"); } - DiscontiguousMatrix(const DiscontiguousMatrix&) = delete; + DiscontiguousMatrix(const DiscontiguousMatrix &) = delete; ~DiscontiguousMatrix() { print_destroyed(this, std::to_string(rows() / m_row_factor) + "(*" diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 40c57df2ac..0c05614e6a 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -521,7 +521,7 @@ TEST_SUBMODULE(class_, m) { // test_exception_rvalue_abort struct PyPrintDestructor { PyPrintDestructor() = default; - PyPrintDestructor(const PyPrintDestructor&) = default; + PyPrintDestructor(const PyPrintDestructor &) = default; ~PyPrintDestructor() { py::print("Print from destructor"); } void throw_something() { throw std::runtime_error("error"); } }; diff --git a/tests/test_class_release_gil_before_calling_cpp_dtor.cpp b/tests/test_class_release_gil_before_calling_cpp_dtor.cpp index 713b5015c2..7349c416bb 100644 --- a/tests/test_class_release_gil_before_calling_cpp_dtor.cpp +++ b/tests/test_class_release_gil_before_calling_cpp_dtor.cpp @@ -22,7 +22,7 @@ struct ProbeType { public: explicit ProbeType(const std::string &unique_key) : unique_key{unique_key} {} - ProbeType(const ProbeType&) = default; + ProbeType(const ProbeType &) = default; ~ProbeType() { RegistryType ® = PyGILState_Check_Results(); diff --git a/tests/test_cross_module_rtti/lib.h b/tests/test_cross_module_rtti/lib.h index e11dd19dcf..2f76043c30 100644 --- a/tests/test_cross_module_rtti/lib.h +++ b/tests/test_cross_module_rtti/lib.h @@ -12,7 +12,7 @@ __pragma(warning(disable : 4251)) class TEST_CROSS_MODULE_RTTI_LIB_EXPORT Base : public std::enable_shared_from_this { public: Base(int a, int b); - Base(const Base&) = default; + Base(const Base &) = default; virtual ~Base() = default; virtual int get() const; diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index d4e83bdc47..96a96c2b18 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -237,7 +237,7 @@ TEST_SUBMODULE(eigen_matrix, m) { public: ReturnTester() { print_created(this); } - ReturnTester(const ReturnTester&) = default; + ReturnTester(const ReturnTester &) = default; ~ReturnTester() { print_destroyed(this); } static Eigen::MatrixXd create() { return Eigen::MatrixXd::Ones(10, 10); } // NOLINTNEXTLINE(readability-const-return-type) diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 3f75f75101..28359c46d4 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -282,7 +282,7 @@ TEST_SUBMODULE(numpy_array, sm) { struct ArrayClass { int data[2] = {1, 2}; ArrayClass() { py::print("ArrayClass()"); } - ArrayClass(const ArrayClass&) = default; + ArrayClass(const ArrayClass &) = default; ~ArrayClass() { py::print("~ArrayClass()"); } }; py::class_(sm, "ArrayClass") diff --git a/tests/test_potentially_slicing_weak_ptr.cpp b/tests/test_potentially_slicing_weak_ptr.cpp index 57de2fa102..7389d9600e 100644 --- a/tests/test_potentially_slicing_weak_ptr.cpp +++ b/tests/test_potentially_slicing_weak_ptr.cpp @@ -12,7 +12,7 @@ namespace potentially_slicing_weak_ptr { template // Using int as a trick to easily generate multiple types. struct VirtBase { virtual ~VirtBase() = default; - VirtBase(const VirtBase&) = delete; + VirtBase(const VirtBase &) = delete; virtual int get_code() { return 100; } }; diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp index 2db5293637..2e98d469f2 100644 --- a/tests/test_smart_ptr.cpp +++ b/tests/test_smart_ptr.cpp @@ -161,7 +161,7 @@ class MyObject4a { print_created(this); pointer_set().insert(this); }; - MyObject4a(const MyObject4a&) = delete; + MyObject4a(const MyObject4a &) = delete; int value; @@ -184,7 +184,7 @@ class MyObject4a { class MyObject4b : public MyObject4a { public: explicit MyObject4b(int i) : MyObject4a(i) { print_created(this); } - MyObject4b(const MyObject4b&) = delete; + MyObject4b(const MyObject4b &) = delete; ~MyObject4b() override { print_destroyed(this); } }; @@ -192,7 +192,7 @@ class MyObject4b : public MyObject4a { class MyObject5 { // managed by huge_unique_ptr public: explicit MyObject5(int value) : value{value} { print_created(this); } - MyObject5(const MyObject5&) = delete; + MyObject5(const MyObject5 &) = delete; ~MyObject5() { print_destroyed(this); } int value; }; @@ -249,7 +249,7 @@ struct SharedFromThisVirt : virtual SharedFromThisVBase {}; // test_move_only_holder struct C { C() { print_created(this); } - C(const C&) = delete; + C(const C &) = delete; ~C() { print_destroyed(this); } }; diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 969af98dec..6084d517df 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -99,7 +99,7 @@ class OptionalProperties { using OptionalEnumValue = OptionalImpl; OptionalProperties() : value(EnumType::kSet) {} - OptionalProperties(const OptionalProperties&) = default; + OptionalProperties(const OptionalProperties &) = default; ~OptionalProperties() { // Reset value to detect use-after-destruction. // This is set to a specific value rather than nullopt to ensure that diff --git a/tests/test_tagbased_polymorphic.cpp b/tests/test_tagbased_polymorphic.cpp index 1d709732f0..8a8a3280c8 100644 --- a/tests/test_tagbased_polymorphic.cpp +++ b/tests/test_tagbased_polymorphic.cpp @@ -17,7 +17,7 @@ struct Animal { // (https://github.com/pybind/pybind11/pull/2016/). virtual ~Animal() = default; - Animal(const Animal&) = delete; + Animal(const Animal &) = delete; // Enum for tag-based polymorphism. enum class Kind { From 8acf6d7a0c4639b53ce4d1df73400a10239ff16b Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sun, 17 Aug 2025 21:11:50 -0600 Subject: [PATCH 07/18] CI fixes --- include/pybind11/detail/type_caster_base.h | 4 ++-- include/pybind11/pybind11.h | 9 +++++---- tests/test_potentially_slicing_weak_ptr.cpp | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 93a7e407c5..013d8e3e85 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -1708,7 +1708,7 @@ class type_caster_base : public type_caster_generic { srcs, return_value_policy::take_ownership, {}, nullptr, nullptr, holder); if (srcs.used_foreign) { // Foreign cast succeeded; release C++ ownership - (void) holder->release(); + (void) holder->release(); // NOLINT(bugprone-unused-return-value) } return ret; } @@ -1724,7 +1724,7 @@ class type_caster_base : public type_caster_generic { ret.dec_ref(); throw error_already_set(); } - (void) sp.release(); + (void) sp.release(); // NOLINT(bugprone-unused-return-value) } template diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index ae10656bc4..ecffaf91ce 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -158,10 +158,11 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel if (!t) { pybind11_fail("Internal error while parsing type signature (1)"); } - if (handle th = detail::get_type_handle(*t, false, true)) { - signature += th.attr("__module__").cast() + "." - + th.attr("__qualname__").cast(); - } else if ((th = detail::global_internals_native_enum_type_map_get_item(*t))) { + handle th = detail::get_type_handle(*t, false, true); + if (!th) { + th = detail::global_internals_native_enum_type_map_get_item(*t); + } + if (th) { signature += th.attr("__module__").cast() + "." + th.attr("__qualname__").cast(); } else if (func_rec->is_new_style_constructor && arg_index == 0) { diff --git a/tests/test_potentially_slicing_weak_ptr.cpp b/tests/test_potentially_slicing_weak_ptr.cpp index 7389d9600e..c1bf36f194 100644 --- a/tests/test_potentially_slicing_weak_ptr.cpp +++ b/tests/test_potentially_slicing_weak_ptr.cpp @@ -11,6 +11,7 @@ namespace potentially_slicing_weak_ptr { template // Using int as a trick to easily generate multiple types. struct VirtBase { + VirtBase() = default; virtual ~VirtBase() = default; VirtBase(const VirtBase &) = delete; virtual int get_code() { return 100; } From 2ecbe8e044a6ddf37a09952be14759df8ecd655c Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 21 Aug 2025 10:56:31 -0600 Subject: [PATCH 08/18] Self review, update pymetabind, resolve TODO --- include/pybind11/cast.h | 8 ++++---- include/pybind11/detail/foreign.h | 4 +--- include/pybind11/detail/pymetabind.h | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index dadbaa5191..09a81fa495 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -876,20 +876,20 @@ struct holder_caster_foreign_helpers { PyObject *o; }; - template - static bool try_shared_from_this(std::enable_shared_from_this *value, + template + static bool try_shared_from_this(std::enable_shared_from_this *value, std::shared_ptr *holder_out) { // object derives from enable_shared_from_this; // try to reuse an existing shared_ptr if one is known if (auto existing = try_get_shared_from_this(value)) { - *holder_out = existing; + *holder_out = std::static_pointer_cast(existing); return true; } return false; } template - static bool try_shared_from_this(type *, std::shared_ptr *) { + static bool try_shared_from_this(void *, std::shared_ptr *) { return false; } diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index 2a52292f1e..1aa7373764 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -289,10 +289,8 @@ PYBIND11_NOINLINE bool foreign_internals::initialize() { self.reset(new pymb_framework{}); self->name = "pybind11 " PYBIND11_ABI_TAG; - // TODO: pybind11 does leak some bindings; there should be a way to - // indicate that (so that eg nanobind can disable its leak detection) - // without promising to leak all bindings self->bindings_usable_forever = 0; + self->leak_safe = 0; self->abi_lang = pymb_abi_lang_cpp; self->abi_extra = PYBIND11_PLATFORM_ABI_ID; self->from_python = foreign_cb_from_python; diff --git a/include/pybind11/detail/pymetabind.h b/include/pybind11/detail/pymetabind.h index d42cd9c7eb..d7bf2628bd 100644 --- a/include/pybind11/detail/pymetabind.h +++ b/include/pybind11/detail/pymetabind.h @@ -6,7 +6,10 @@ * This functionality is intended to be used by the framework itself, * rather than by users of the framework. * - * This is version 0.1 of pymetabind. Changelog: + * This is version 0.1+dev of pymetabind. Changelog: + * + * Unreleased: Fix typo in Py_GIL_DISABLED. Add pymb_framework::leak_safe. + * Add casts from PyTypeObject* to PyObject* where needed. * * Version 0.1: Initial draft. ABI may change without warning while we * 2025-08-16 prove out the concept. Please wait for a 1.0 release @@ -195,7 +198,7 @@ struct pymb_registry { #endif }; -#if defined(Py_GIL_DISALED) +#if defined(Py_GIL_DISABLED) inline void pymb_lock_registry(struct pymb_registry* registry) { PyMutex_Lock(®istry->mutex); } @@ -250,8 +253,15 @@ struct pymb_framework { // this framework's bindings in free-threaded builds. uint8_t bindings_usable_forever; + // Does this framework reliably deallocate all of its type and function + // objects by the time the Python interpreter is finalized, in the absence + // of bugs in user code? If not, it might cause leaks of other frameworks' + // types or functions, via attributes or default argument values for + // this framework's leaked objects. + uint8_t leak_safe; + // Reserved for future extensions. Set to 0. - uint8_t reserved[3]; + uint8_t reserved[2]; // The language to which this framework provides bindings: one of the // `pymb_abi_lang` enumerators. From d5727c6bc08f76c3c35b2815b90b7ded89d79cbc Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 13 Sep 2025 16:55:02 -0600 Subject: [PATCH 09/18] Merge upstream pymetabind changes --- include/pybind11/detail/pymetabind.h | 776 ++++++++++++++++++++------- 1 file changed, 575 insertions(+), 201 deletions(-) diff --git a/include/pybind11/detail/pymetabind.h b/include/pybind11/detail/pymetabind.h index d7bf2628bd..a4a7cca43f 100644 --- a/include/pybind11/detail/pymetabind.h +++ b/include/pybind11/detail/pymetabind.h @@ -6,10 +6,26 @@ * This functionality is intended to be used by the framework itself, * rather than by users of the framework. * - * This is version 0.1+dev of pymetabind. Changelog: + * This is version 0.2+dev of pymetabind. Changelog: * - * Unreleased: Fix typo in Py_GIL_DISABLED. Add pymb_framework::leak_safe. + * Unreleased: Don't do a Py_DECREF in `pymb_remove_framework` since the + * interpreter might already be finalized at that point. + * Revamp binding lifetime logic. Add `remove_local_binding` + * and `free_local_binding` callbacks. + * Add `pymb_framework::registry` and use it to simplify + * the signatures of `pymb_remove_framework`, + * `pymb_add_binding`, and `pymb_remove_binding`. + * + * Version 0.2: Use a bitmask for `pymb_framework::flags` and add leak_safe + * 2025-09-11 flag. Change `translate_exception` to be non-throwing. * Add casts from PyTypeObject* to PyObject* where needed. + * Fix typo in Py_GIL_DISABLED. Add noexcept to callback types. + * Rename `hook` -> `link` in linked list nodes. + * Use `static inline` linkage in C. Free registry on exit. + * Clear list hooks when adding frameworks/bindings in case + * the user didn't zero-initialize. Avoid abi_extra string + * comparisons if the strings are already pointer-equal. + * Add `remove_foreign_framework` callback. * * Version 0.1: Initial draft. ABI may change without warning while we * 2025-08-16 prove out the concept. Please wait for a 1.0 release @@ -43,16 +59,25 @@ #include #include +#include #if !defined(PY_VERSION_HEX) # error You must include Python.h before this header #endif +// `inline` in C implies a promise to provide an out-of-line definition +// elsewhere; in C++ it does not. +#ifdef __cplusplus +# define PYMB_INLINE inline +#else +# define PYMB_INLINE static inline +#endif + /* * There are two ways to use this header file. The default is header-only style, - * where all functions are defined as `inline`. If you want to emit functions - * as non-inline, perhaps so you can link against them from non-C/C++ code, - * then do the following: + * where all functions are defined as `inline` (C++) / `static inline` (C). + * If you want to emit functions as non-inline, perhaps so you can link against + * them from non-C/C++ code, then do the following: * - In every compilation unit that includes this header, `#define PYMB_FUNC` * first. (The `PYMB_FUNC` macro will be expanded in place of the "inline" * keyword, so you can also use it to add any other declaration attributes @@ -62,11 +87,14 @@ * compilation unit that doesn't request `PYMB_DECLS_ONLY`. */ #if !defined(PYMB_FUNC) -#define PYMB_FUNC inline +#define PYMB_FUNC PYMB_INLINE #endif #if defined(__cplusplus) +#define PYMB_NOEXCEPT noexcept extern "C" { +#else +#define PYMB_NOEXCEPT #endif /* @@ -135,11 +163,11 @@ struct pymb_list { struct pymb_list_node head; }; -inline void pymb_list_init(struct pymb_list* list) { +PYMB_INLINE void pymb_list_init(struct pymb_list* list) { list->head.prev = list->head.next = &list->head; } -inline void pymb_list_unlink(struct pymb_list_node* node) { +PYMB_INLINE void pymb_list_unlink(struct pymb_list_node* node) { if (node->next) { node->next->prev = node->prev; node->prev->next = node->next; @@ -147,8 +175,8 @@ inline void pymb_list_unlink(struct pymb_list_node* node) { } } -inline void pymb_list_append(struct pymb_list* list, - struct pymb_list_node* node) { +PYMB_INLINE void pymb_list_append(struct pymb_list* list, + struct pymb_list_node* node) { pymb_list_unlink(node); struct pymb_list_node* tail = list->head.prev; tail->next = node; @@ -157,10 +185,14 @@ inline void pymb_list_append(struct pymb_list* list, node->next = &list->head; } +PYMB_INLINE int pymb_list_is_empty(struct pymb_list* list) { + return list->head.next == &list->head; +} + #define PYMB_LIST_FOREACH(type, name, list) \ for (type name = (type) (list).head.next; \ name != (type) &(list).head; \ - name = (type) name->hook.next) + name = (type) name->link.next) /* * The registry holds information about all the interoperable binding @@ -188,8 +220,14 @@ struct pymb_registry { // Linked list of registered `pymb_binding` structures struct pymb_list bindings; + // Heap-allocated PyMethodDef for bound type weakref callback + PyMethodDef* weakref_callback_def; + // Reserved for future extensions; currently set to 0 - uint32_t reserved; + uint16_t reserved; + + // Set to true when the capsule that points to this registry is destroyed + uint8_t deallocate_when_empty; #if defined(Py_GIL_DISABLED) // Mutex guarding accesses to `frameworks` and `bindings`. @@ -199,19 +237,41 @@ struct pymb_registry { }; #if defined(Py_GIL_DISABLED) -inline void pymb_lock_registry(struct pymb_registry* registry) { +PYMB_INLINE void pymb_lock_registry(struct pymb_registry* registry) { PyMutex_Lock(®istry->mutex); } -inline void pymb_unlock_registry(struct pymb_registry* registry) { +PYMB_INLINE void pymb_unlock_registry(struct pymb_registry* registry) { PyMutex_Unlock(®istry->mutex); } #else -inline void pymb_lock_registry(struct pymb_registry*) {} -inline void pymb_unlock_registry(struct pymb_registry*) {} +PYMB_INLINE void pymb_lock_registry(struct pymb_registry* registry) { + (void) registry; +} +PYMB_INLINE void pymb_unlock_registry(struct pymb_registry* registry) { + (void) registry; +} #endif struct pymb_binding; +/* Flags for a `pymb_framework` */ +enum pymb_framework_flags { + // Does this framework guarantee that its `pymb_binding` structures remain + // valid to use for the lifetime of the Python interpreter process once + // they have been linked into the lists in `pymb_registry`? Setting this + // flag reduces the number of atomic operations needed to work with + // this framework's bindings in free-threaded builds. + pymb_framework_bindings_usable_forever = 0x0001, + + // Does this framework reliably deallocate all of its type and function + // objects by the time the Python interpreter is finalized, in the absence + // of bugs in user code? If not, it might cause leaks of other frameworks' + // types or functions, via attributes or default argument values for + // this framework's leaked objects. Other frameworks can suppress their + // leak warnings (if so equipped) when a non-`leak_safe` framework is added. + pymb_framework_leak_safe = 0x0002, +}; + /* * Information about one framework that has registered itself with pymetabind. * "Framework" here refers to a set of bindings that are natively mutually @@ -229,7 +289,7 @@ struct pymb_binding; * and unmodified (except as documented below) until the Python interpreter * is finalized. After finalization, such as in a `Py_AtExit` handler, if * all bindings have been removed already, you may optionally clean up by - * calling `pymb_list_unlink(&framework->hook)` and then deallocating the + * calling `pymb_remove_framework()` and then deallocating the * `pymb_framework` structure. * * All fields of this structure are set before it is made visible to other @@ -238,27 +298,22 @@ struct pymb_binding; * individual documentation. */ struct pymb_framework { - // Hook by which this structure is linked into the list of + // Links to the previous and next framework in the list of // `pymb_registry::frameworks`. May be modified as other frameworks are // added; protected by the `pymb_registry::mutex` in free-threaded builds. - struct pymb_list_node hook; + struct pymb_list_node link; + + // Link to the `pymb_registry` that this framework is registered with. + // Filled in by `pymb_add_framework()`. + struct pymb_registry* registry; // Human-readable description of this framework, as a NUL-terminated string const char* name; - // Does this framework guarantee that its `pymb_binding` structures remain - // valid to use for the lifetime of the Python interpreter process once - // they have been linked into the lists in `pymb_registry`? Setting this - // to true reduces the number of atomic operations needed to work with - // this framework's bindings in free-threaded builds. - uint8_t bindings_usable_forever; - - // Does this framework reliably deallocate all of its type and function - // objects by the time the Python interpreter is finalized, in the absence - // of bugs in user code? If not, it might cause leaks of other frameworks' - // types or functions, via attributes or default argument values for - // this framework's leaked objects. - uint8_t leak_safe; + // Flags for this framework, a combination of `enum pymb_framework_flags`. + // Undefined flags must be set to zero to allow for future + // backward-compatible extensions. + uint16_t flags; // Reserved for future extensions. Set to 0. uint8_t reserved[2]; @@ -291,8 +346,8 @@ struct pymb_framework { // The function pointers below allow other frameworks to interact with // bindings provided by this framework. They are constant after construction - // and, except for `translate_exception()`, must not throw C++ exceptions. - // Unless otherwise documented, they must not be NULL. + // and must not throw C++ exceptions. Unless otherwise documented, + // they must not be NULL. // Extract a C/C++/etc object from `pyobj`. The desired type is specified by // providing a `pymb_binding*` for some binding that belongs to this @@ -312,7 +367,13 @@ struct pymb_framework { // `keep_referenced` should incref its `obj` immediately and remember // that it should be decref'ed later, for no net change in refcount. // This is an abstraction around something like the cleanup_list in - // nanobind or loader_life_support in pybind11. + // nanobind or loader_life_support in pybind11. The pointer returned by + // `from_python` may be invalidated once the `keep_referenced` references + // are dropped. If you're converting a function argument, you should keep + // any `keep_referenced` references alive until the function returns. + // If you're converting for some other purpose, you probably want to make + // a copy of the object to which `from_python`'s return value points before + // you drop the references. // // On free-threaded builds, callers must ensure that the `binding` is not // destroyed during a call to `from_python`. The requirements for this are @@ -321,7 +382,7 @@ struct pymb_framework { PyObject* pyobj, uint8_t convert, void (*keep_referenced)(void* ctx, PyObject* obj), - void* keep_referenced_ctx); + void* keep_referenced_ctx) PYMB_NOEXCEPT; // Wrap the C/C++/etc object `cobj` into a Python object using the given // return value policy. The type is specified by providing a `pymb_binding*` @@ -341,7 +402,7 @@ struct pymb_framework { PyObject* (*to_python)(struct pymb_binding* binding, void* cobj, enum pymb_rv_policy rvp, - PyObject* parent); + PyObject* parent) PYMB_NOEXCEPT; // Request that a PyObject reference be dropped, or that a callback // be invoked, when `nurse` is destroyed. `nurse` should be an object @@ -351,148 +412,252 @@ struct pymb_framework { // or -1 and sets the Python error indicator on error. // // No synchronization is required to call this method. - int (*keep_alive)(PyObject* nurse, void* payload, void (*cb)(void*)); - - // Attempt to translate a C++ exception known to this framework to Python. - // This should translate only framework-specific exceptions or user-defined - // exceptions that were registered with the framework, not generic - // ones such as `std::exception`. If successful, return normally with the - // Python error indicator set; otherwise, reraise the provided exception. - // `eptr` should be cast to `const std::exception_ptr* eptr` before use. - // This function pointer may be NULL if this framework does not provide - // C++ exception translation. + int (*keep_alive)(PyObject* nurse, + void* payload, + void (*cb)(void*)) PYMB_NOEXCEPT; + + // Attempt to translate the native exception `eptr` into a Python exception. + // If `abi_lang` is C++, then `eptr` should be cast to `std::exception_ptr*` + // before use; semantics for other languages have not been defined yet. This + // should translate only framework-specific exceptions or user-defined + // exceptions that were registered with the framework, not generic ones + // such as `std::exception`. If translation succeeds, return 1 with the + // Python error indicator set; otherwise, return 0. An exception may be + // converted into a different exception by modifying `*eptr` and returning + // zero. This function pointer may be NULL if this framework does not + // provide exception translation. // // No synchronization is required to call this method. - void (*translate_exception)(const void* eptr); + int (*translate_exception)(void* eptr) PYMB_NOEXCEPT; + + // Notify this framework that one of its own bindings is being removed. + // This will occur synchronously from within a call to + // `pymb_remove_binding()`. Don't free the binding yet; wait for a later + // call to `free_local_binding`. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*remove_local_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; + + // Request this framework to free one of its own binding structures. + // A call to `pymb_remove_binding()` will eventually result in a call to + // this method, once pymetabind can prove no one is concurrently using the + // binding anymore. + // + // No synchronization is required to call this method. + void (*free_local_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; // Notify this framework that some other framework published a new binding. // This call will be made after the new binding has been linked into the // `pymb_registry::bindings` list. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*add_foreign_binding)(struct pymb_binding* binding); + void (*add_foreign_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; // Notify this framework that some other framework is about to remove // a binding. This call will be made after the binding has been removed // from the `pymb_registry::bindings` list. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*remove_foreign_binding)(struct pymb_binding* binding); + void (*remove_foreign_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; // Notify this framework that some other framework came into existence. // This call will be made after the new framework has been linked into the // `pymb_registry::frameworks` list and before it adds any bindings. // // The `pymb_registry::mutex` or GIL will be held when calling this method. - void (*add_foreign_framework)(struct pymb_framework* framework); + void (*add_foreign_framework)(struct pymb_framework* framework) PYMB_NOEXCEPT; - // There is no remove_foreign_framework(); the interpreter has - // already been finalized at that point, so there's nothing for the - // callback to do. + // Notify this framework that some other framework is being destroyed. + // This call will be made after the framework has been removed from the + // `pymb_registry::frameworks` list. + // + // This can only occur during interpreter finalization, so no + // synchronization is required. It might occur very late in interpreter + // finalization, such as from a Py_AtExit handler, so it shouldn't + // execute Python code. + void (*remove_foreign_framework)(struct pymb_framework* framework) PYMB_NOEXCEPT; }; /* * Information about one type binding that belongs to a registered framework. * + * ### Creating bindings + * * A framework that binds some type and wants to allow other frameworks to * work with objects of that type must create a `pymb_binding` structure for * the type. This can be allocated in any way that the framework prefers (e.g., - * on the heap or within the type object). Once filled out, the binding - * structure should be passed to `pymb_add_binding()`. If the Python type object - * underlying the binding is to be deallocated, a `pymb_remove_binding()` call - * must be made, and the `pymb_binding` structure cannot be deallocated until - * `pymb_remove_binding()` returns. The call to `pymb_remove_binding()` - * must occur *during* deallocation of the binding's Python type object, i.e., - * at a time when `Py_REFCNT(pytype) == 0` but the storage for `pytype` is not - * yet eligible to be reused for another object. Many frameworks use a custom - * metaclass, and can add the call to `pymb_remove_binding()` from the metaclass - * `tp_dealloc`; those that don't can use a weakref callback on the type object - * instead. The constraint on destruction timing allows `pymb_try_ref_binding()` - * to temporarily prevent the binding's destruction by incrementing the type - * object's reference count. + * on the heap or within the type object). Any fields without a meaningful + * value must be zero-filled. Once filled out, the binding structure should be + * passed to `pymb_add_binding()`. This will advertise the binding to other + * frameworks' `add_foreign_binding` hooks. It also creates a capsule object + * that points to the `pymb_binding` structure, and stores this capsule in the + * bound type's dict as the attribute "__pymetabind_binding__". + * The intended use of both of these is discussed later on in this comment. + * + * ### Removing bindings + * + * From a user perspective, a binding can be removed for either of two reasons: + * + * - its capsule was destroyed, such as by `del MyType.__pymetabind_binding__` + * - its type object is being finalized + * + * These both result in a call to `pymb_remove_binding()` that begins the + * removal process, but you should not call that function yourself, except + * from a metatype `tp_finalize` as described below. Some time after the call + * to `pymb_remove_binding()`, pymetabind will call the binding's framework's + * `free_local_binding` hook to indicate that it's safe to actually free the + * `pymb_binding` structure. + * + * By default, pymetabind detects the finalization of a binding's type object + * by creating a weakref to the type object with an appropriate callback. This + * works fine, but requires several objects to be allocated, so it is not ideal + * from a memory overhead perspective. If you control the bound type's metatype, + * you can reduce this overhead by modifying the metatype's `tp_finalize` slot + * to call `pymb_remove_binding()`. If you tell pymetabind that you have done + * so, using the `tp_finalize_will_remove` argument to `pymb_add_binding()`, + * then pymetabind won't need to create the weakref and its callback. + * + * ### Removing bindings: the gory details + * + * The implementation of the removal process is somewhat complex in order to + * protect other threads that might be concurrently using a binding in + * free-threaded builds. `pymb_remove_binding()` stops new uses of the binding + * from beginning, by notifying other frameworks' `remove_foreign_binding` + * hooks and changing the binding capsule so `pymb_get_binding()` won't work. + * Existing uses might be ongoing though, so we must wait for them to complete + * before freeing the binding structure. The technique we choose is to wait for + * the next (or current) garbage collection to finish. GC stops all threads + * before it scans the heap. An attached thread state (one that can call + * CPython API functions) can't be stopped without its consent, so GC will + * wait for it to detach. A thread state can only become detached explicitly + * (e.g. Py_BEGIN_ALLOW_THREADS) or in the bytecode interpreter. As long as + * foreign frameworks don't hold `pymb_binding` pointers across calls into + * the bytecode interpreter in places their `remove_foreign_binding` hook + * can't see, this technique avoids use-after-free without introducing any + * contention on a shared atomic in the binding object. + * + * One pleasant aspect of this scheme: due to their use of deferred reference + * counting, type objects in free-threaded Python can only be freed during a + * GC pass. There is even a stop-all-threads (to check for resurrected objects) + * in between when GC executes finalizers and when it actually destroys the + * garbage. This winds up letting us obtain the "wait for next GC pause before + * freeing the binding" behavior very cheaply when the binding is being removed + * due to the deletion of its type. + * + * On non-free-threaded Python builds, none of the above is a concern, and + * `pymb_remove_binding()` can synchronously free the `pymb_binding` structure. + * + * ### Keeping track of other frameworks' bindings + * + * In order to work with Python objects bound by another framework, yours + * must be able to locate a `pymb_binding` structure for that type. It is + * anticipated that most frameworks will maintain their own private + * type-to-binding maps, which they can keep up-to-date via their + * `add_foreign_binding` and `remove_foreign_binding` hooks. It is important + * to think carefully about how to design the synchronization for these maps + * so that lookups do not return pointers to bindings that may have been + * deallocated. The remainder of this section provides some suggestions. + * + * The recommended way to handle synchronization is to protect your type lookup + * map with a readers/writer lock. In your `remove_foreign_binding` hook, + * obtain a write lock, and hold it while removing the corresponding entry from + * the map. Before performing a type lookup, obtain a read lock. If the lookup + * succeeds, you can release the read lock and (due to the two-phase removal + * process described above) continue to safely use the binding for as long as + * your Python thread state remains attached. It is important not to hold the + * read lock while executing arbitrary Python code, since a deadlock would + * result if the binding were removed (requiring a write lock) while the read + * lock were held. Note that `pymb_framework::from_python` for many popular + * frameworks can execute arbitrary Python code to perform implicit conversions. + * + * If you're trying multiple bindings for an operation, one option is to copy + * all their pointers to temporary storage before releasing the read lock. + * (While concurrent updates may modify the data structure, the pymb_binding + * structures it points to will remain valid for long enough.) If you prefer + * to avoid the copy by unlocking for each attempt and then relocking to + * advance to the next binding, be sure to consider the possibility that your + * iterator might have been invalidated due to a concurrent update while you + * weren't holding the lock. + * + * The lock on a single shared type lookup map is a contention bottleneck, + * especially if you don't have a readers/writer lock and wish to get by with + * an ordinary mutex. To improve performance, you can give each thread its + * own lookup map, and require `remove_foreign_binding` to update all of them. + * As long as the per-thread maps are always visited in a consistent order + * when removing a binding, the splitting shouldn't introduce new deadlocks. + * Since each thread can have a separate mutex for its separate map, contention + * occurs only when bindings are being added or removed, which is much less + * common than using them. + * + * ### Using the binding capsule * * Each Python type object for which a `pymb_binding` exists will have an * attribute "__pymetabind_binding__" whose value is a capsule object * that contains the `pymb_binding` pointer under the name "pymetabind_binding". - * The attribute is set during `pymb_add_binding()`. This is provided to allow: + * The attribute is set during `pymb_add_binding()`, and is used by + * `pymb_get_binding()` to map a type object to a binding. The capsule allows: + * * - Determining which framework to call for a foreign `keep_alive` operation + * * - Locating `pymb_binding` objects for types written in a different language * than yours (where you can't look up by the `pymb_binding::native_type`), * so that you can work with their contents using non-Python-specific * cross-language support + * * - Extracting the native object from a Python object without being too picky * about what type it is (risky, but maybe you have out-of-band information * that shows it's safe) + * * The preferred mechanism for same-language object access is to maintain a * hashtable keyed on `pymb_binding::native_type` and look up the binding for * the type you want/have. Compared to reading the capsule, this better * supports inheritance, to-Python conversions, and implicit conversions, and * it's probably also faster depending on how it's implemented. * + * ### Types with multiple bindings + * * It is valid for multiple frameworks to claim (in separate bindings) the - * same C/C++ type, or even the same Python type. (A case where multiple - * frameworks would bind the same Python type is if one is acting as an - * extension to the other, such as to support extracting pointers to - * non-primary base classes when the base framework doesn't think about - * such things.) If multiple frameworks claim the same Python type, then each - * new registrant will replace the "__pymetabind_binding__" capsule and there - * is no way to locate the other bindings from the type object. + * same C/C++ type. This supports cases where a common vocabulary type is + * bound separately in mulitple extensions in the same process. Frameworks + * are encouraged to try all registered bindings for the target type when + * they perform from-Python conversions. * - * All fields of this structure are set before it is made visible to other - * threads and then never changed, so they don't need locking to access. - * However, on free-threaded builds it is necessary to validate that the type - * object is not partway through being destroyed before you use the binding, - * and prevent such destruction from beginning until you're done. To do so, - * call `pymb_try_ref_binding()`; if it returns false, don't use the binding, - * else use it and then call `pymb_unref_binding()` when done. - * (On non-free-threaded builds, these do incref/decref to prevent destruction - * of the type from starting, but can't fail because there's no *concurrent* - * destruction hazard.) - * - * In order to work with one framework's Python objects of a certain type, other - * frameworks must be able to locate a `pymb_binding` structure for that type. - * It is expected that they will maintain their own type-to-binding maps, which - * they can keep up-to-date via their `pymb_framework::add_foreign_binding` and - * `pymb_framework::remove_foreign_binding` hooks. It is important to think very - * carefully about how to design the synchronization for these maps so that - * lookups do not return pointers to bindings that have been deallocated. - * The remainder of this comment provides some suggestions. + * If multiple frameworks claim the same Python type, the last one will + * typically win, since there is only one "__pymetabind_binding__" attribute + * on the type object and a binding is removed when its capsule is no longer + * referenced. If you're trying to do something unusual like wrapping another + * framework's binding to provide additional features, you can stash the + * extra binding(s) under a different attribute name. pymetabind never uses + * the "__pymetabind_binding__" attribute to locate the binding for its own + * purposes; it's used only to fulfill calls to `pymb_get_binding()`. * - * The recommended way to handle synchronization is to protect your type lookup - * map with a readers/writer lock. In your `remove_foreign_binding` hook, - * obtain a write lock, and hold it while removing the corresponding entry from - * the map. Before performing a type lookup, obtain a read lock. If the lookup - * succeeds, call `pymb_try_ref_binding()` on the resulting binding before - * you release your read lock. Since the binding structure can't be deallocated - * until all `remove_foreign_binding` hooks have returned, this scheme provides - * effective protection. It is important not to hold the read lock while - * executing arbitrary Python code, since a deadlock would result if the type - * object is deallocated (requiring a write lock) while the read lock were held. - * Note that `pymb_framework::from_python` for many popular frameworks is - * capable of executing arbitrary Python code to perform implicit conversions. + * ### Synchronization * - * The lock on a single shared type lookup map is a contention bottleneck, - * especially if you don't have a readers/writer lock and wish to get by with - * an ordinary mutex. To improve performance, you can give each thread its - * own lookup map, and require `remove_foreign_binding` to update all of them. - * As long as the per-thread maps are always visited in a consistent order - * when removing a binding, the splitting shouldn't introduce new deadlocks. - * Since each thread has a separate mutex for its separate map, contention - * occurs only when bindings are being added or removed, which is much less - * common than using them. + * Most fields of this structure are set before it is made visible to other + * threads and then never changed, so they don't need locking to access. The + * `link` and `capsule` are protected by the registry lock. */ struct pymb_binding { - // Hook by which this structure is linked into the list of + // Links to the previous and next bindings in the list of // `pymb_registry::bindings` - struct pymb_list_node hook; + struct pymb_list_node link; // The framework that provides this binding struct pymb_framework* framework; + // Borrowed reference to the capsule object that refers to this binding. + // Becomes NULL in pymb_remove_binding(). + PyObject* capsule; + // Python type: you will get an instance of this type from a successful // call to `framework::from_python()` that passes this binding PyTypeObject* pytype; + // Strong reference to a weakref to `pytype`; its callback will prompt + // us to remove the binding. May be NULL if Py_TYPE(pytype)->tp_finalize + // will take care of that. + PyObject* pytype_wr; + // The native identifier for this type in `framework->abi_lang`, if that is // a concept that exists in that language. See the documentation of // `enum pymb_abi_lang` for specific per-language semantics. @@ -500,6 +665,9 @@ struct pymb_binding { // The way that this type would be written in `framework->abi_lang` source // code, as a NUL-terminated byte string without struct/class/enum words. + // If `framework->abi_lang` uses name mangling, this is the demangled, + // human-readable name. C++ users should note that the result of + // `typeid(x).name()` will need platform-specific alteration to produce one. // Examples: "Foo", "Bar::Baz", "std::vector >" const char* source_name; @@ -509,27 +677,56 @@ struct pymb_binding { void* context; }; -/* - * Users of non-C/C++ languages are welcome to replicate the logic of these - * inline functions rather than calling them. Their implementations are - * considered part of the ABI. - */ - PYMB_FUNC struct pymb_registry* pymb_get_registry(); PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, struct pymb_framework* framework); -PYMB_FUNC void pymb_remove_framework(struct pymb_registry* registry, - struct pymb_framework* framework); -PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, - struct pymb_binding* binding); -PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, - struct pymb_binding* binding); -PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding); -PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding); +PYMB_FUNC void pymb_remove_framework(struct pymb_framework* framework); +PYMB_FUNC void pymb_add_binding(struct pymb_binding* binding, + int tp_finalize_will_remove); +PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding); PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); #if !defined(PYMB_DECLS_ONLY) +PYMB_INLINE void pymb_registry_free(struct pymb_registry* registry) { + assert(pymb_list_is_empty(®istry->bindings) && + "some framework was removed before its bindings"); + free(registry->weakref_callback_def); + free(registry); +} + +PYMB_FUNC void pymb_registry_capsule_destructor(PyObject* capsule) { + struct pymb_registry* registry = + (struct pymb_registry*) PyCapsule_GetPointer( + capsule, "pymetabind_registry"); + if (!registry) { + PyErr_WriteUnraisable(capsule); + return; + } + registry->deallocate_when_empty = 1; + if (pymb_list_is_empty(®istry->frameworks)) { + pymb_registry_free(registry); + } +} + +PYMB_FUNC PyObject* pymb_weakref_callback(PyObject* self, PyObject* weakref) { + // self is bound using PyCFunction_New to refer to a capsule that contains + // the binding pointer (not the binding->capsule; this one has no dtor). + // `weakref` is the weakref (to the bound type) that expired. + if (!PyWeakref_CheckRefExact(weakref) || !PyCapsule_CheckExact(self)) { + PyErr_BadArgument(); + return NULL; + } + struct pymb_binding* binding = + (struct pymb_binding*) PyCapsule_GetPointer( + self, "pymetabind_binding"); + if (!binding) { + return NULL; + } + pymb_remove_binding(binding); + Py_RETURN_NONE; +} + /* * Locate an existing `pymb_registry`, or create a new one if necessary. * Returns a pointer to it, or NULL with the CPython error indicator set. @@ -560,13 +757,32 @@ PYMB_FUNC struct pymb_registry* pymb_get_registry() { if (registry) { pymb_list_init(®istry->frameworks); pymb_list_init(®istry->bindings); - capsule = PyCapsule_New(registry, "pymetabind_registry", NULL); - int rv = capsule ? PyDict_SetItem(dict, key, capsule) : -1; - Py_XDECREF(capsule); - if (rv != 0) { + registry->deallocate_when_empty = 0; + + // C doesn't allow inline functions to declare static variables, + // so allocate this on the heap + PyMethodDef* def = (PyMethodDef*) calloc(1, sizeof(PyMethodDef)); + if (!def) { free(registry); - registry = NULL; + PyErr_NoMemory(); + Py_DECREF(key); + return NULL; } + def->ml_name = "pymetabind_weakref_callback"; + def->ml_meth = pymb_weakref_callback; + def->ml_flags = METH_O; + def->ml_doc = NULL; + registry->weakref_callback_def = def; + + // Attach a destructor so the registry memory is released at teardown + capsule = PyCapsule_New(registry, "pymetabind_registry", + pymb_registry_capsule_destructor); + if (!capsule) { + free(registry); + } else if (PyDict_SetItem(dict, key, capsule) == -1) { + registry = NULL; // will be deallocated by capsule destructor + } + Py_XDECREF(capsule); } else { PyErr_NoMemory(); } @@ -581,21 +797,21 @@ PYMB_FUNC struct pymb_registry* pymb_get_registry() { */ PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, struct pymb_framework* framework) { -#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030e0000 - assert(framework->bindings_usable_forever && - "Free-threaded removal of bindings requires PyUnstable_TryIncRef(), " - "which was added in CPython 3.14"); -#endif + // Defensive: ensure hook is clean before first list insertion to avoid UB + framework->link.next = NULL; + framework->link.prev = NULL; + framework->registry = registry; pymb_lock_registry(registry); PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { // Intern `abi_extra` strings so they can be compared by pointer if (other->abi_extra && framework->abi_extra && - 0 == strcmp(other->abi_extra, framework->abi_extra)) { + (other->abi_extra == framework->abi_extra || + strcmp(other->abi_extra, framework->abi_extra) == 0)) { framework->abi_extra = other->abi_extra; break; } } - pymb_list_append(®istry->frameworks, &framework->hook); + pymb_list_append(®istry->frameworks, &framework->link); PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { if (other != framework) { other->add_foreign_framework(framework); @@ -603,94 +819,252 @@ PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, } } PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { - if (binding->framework != framework && pymb_try_ref_binding(binding)) { + if (binding->framework != framework) { framework->add_foreign_binding(binding); - pymb_unref_binding(binding); } } pymb_unlock_registry(registry); } -/* Add a new binding to the given registry */ -PYMB_FUNC void pymb_add_binding(struct pymb_registry* registry, - struct pymb_binding* binding) { -#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030e0000 - PyUnstable_EnableTryIncRef((PyObject *) binding->pytype); +/* + * Remove a framework from the registry it was added to. + * + * This may only be called during Python interpreter finalization. Rationale: + * other frameworks might be maintaining an entry for the removed one in their + * exception translator lists, and supporting concurrent removal of exception + * translators would add undesirable synchronization overhead to the handling + * of every exception. At finalization time there are no more threads. + * + * Once this function returns, you can free the framework structure. + * + * If a framework never removes itself, it must not claim to be `leak_safe`. + */ +PYMB_FUNC void pymb_remove_framework(struct pymb_framework* framework) { + struct pymb_registry* registry = framework->registry; + + // No need for registry lock/unlock since there are no more threads + pymb_list_unlink(&framework->link); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + other->remove_foreign_framework(framework); + } + + // Destroy registry if capsule is gone and this was the last framework + if (registry->deallocate_when_empty && + pymb_list_is_empty(®istry->frameworks)) { + pymb_registry_free(registry); + } +} + +PYMB_FUNC void pymb_binding_capsule_remove(PyObject* capsule) { + struct pymb_binding* binding = + (struct pymb_binding*) PyCapsule_GetPointer( + capsule, "pymetabind_binding"); + if (!binding) { + PyErr_WriteUnraisable(capsule); + return; + } + pymb_remove_binding(binding); +} + +/* + * Add a new binding for `binding->framework`. If `tp_finalize_will_remove` is + * nonzero, the caller guarantees that `Py_TYPE(binding->pytype).tp_finalize` + * will call `pymb_remove_binding()`; this saves some allocations compared + * to pymetabind needing to figure out when the type is destroyed on its own. + * See the comment on `pymb_binding` for more details. + */ +PYMB_FUNC void pymb_add_binding(struct pymb_binding* binding, + int tp_finalize_will_remove) { + // Defensive: ensure hook is clean before first list insertion to avoid UB + binding->link.next = NULL; + binding->link.prev = NULL; + + binding->pytype_wr = NULL; + binding->capsule = NULL; + + struct pymb_registry* registry = binding->framework->registry; + if (!tp_finalize_will_remove) { + // Different capsule than the binding->capsule, so that the callback + // doesn't keep the binding alive + PyObject* sub_capsule = PyCapsule_New(binding, "pymetabind_binding", + NULL); + if (!sub_capsule) { + goto error; + } + PyObject* callback = PyCFunction_New(registry->weakref_callback_def, + sub_capsule); + Py_DECREF(sub_capsule); // ownership transferred to callback + if (!callback) { + goto error; + } + binding->pytype_wr = PyWeakref_NewRef((PyObject *) binding->pytype, + callback); + Py_DECREF(callback); // ownership transferred to weakref + if (!binding->pytype_wr) { + goto error; + } + } else { +#if defined(Py_GIL_DISABLED) + // No callback needed in this case, but we still do need the weakref + // so that pymb_remove_binding() can tell if the type is being + // finalized or not. + binding->pytype_wr = PyWeakref_NewRef((PyObject *) binding->pytype, + NULL); + if (!binding->pytype_wr) { + goto error; + } #endif - PyObject* capsule = PyCapsule_New(binding, "pymetabind_binding", NULL); - int rv = -1; - if (capsule) { - rv = PyObject_SetAttrString((PyObject *) binding->pytype, - "__pymetabind_binding__", capsule); - Py_DECREF(capsule); } - if (rv != 0) { - PyErr_WriteUnraisable((PyObject *) binding->pytype); + + binding->capsule = PyCapsule_New(binding, "pymetabind_binding", + pymb_binding_capsule_remove); + if (!binding->capsule) { + goto error; + } + if (PyObject_SetAttrString((PyObject *) binding->pytype, + "__pymetabind_binding__", + binding->capsule) != 0) { + Py_CLEAR(binding->capsule); + goto error; } + Py_DECREF(binding->capsule); // keep only a borrowed reference + pymb_lock_registry(registry); - pymb_list_append(®istry->bindings, &binding->hook); + pymb_list_append(®istry->bindings, &binding->link); PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { if (other != binding->framework) { other->add_foreign_binding(binding); } } pymb_unlock_registry(registry); + return; + + error: + PyErr_WriteUnraisable((PyObject *) binding->pytype); + Py_XDECREF(binding->pytype_wr); + binding->framework->free_local_binding(binding); } +#if defined(Py_GIL_DISABLED) +PYMB_FUNC void pymb_binding_capsule_destroy(PyObject* capsule) { + struct pymb_binding* binding = + (struct pymb_binding*) PyCapsule_GetPointer( + capsule, "pymetabind_binding"); + if (!binding) { + PyErr_WriteUnraisable(capsule); + return; + } + Py_CLEAR(binding->pytype_wr); + binding->framework->free_local_binding(binding); +} +#endif + /* - * Remove a binding from the given registry. This must be called during - * deallocation of the `binding->pytype`, such that its reference count is - * zero but still accessible. Once this function returns, you can free the - * binding structure. + * Remove a binding from the registry it was added to. Don't call this yourself, + * except from the tp_finalize slot of a binding's type's metatype. + * The user-servicable way to remove a binding from a still-alive type is to + * delete the capsule. The binding structure will eventually be freed by calling + * `binding->framework->free_local_binding(binding)`. */ -PYMB_FUNC void pymb_remove_binding(struct pymb_registry* registry, - struct pymb_binding* binding) { +PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding) { + struct pymb_registry* registry = binding->framework->registry; + + // Since we need to obtain it anyway, use the registry lock to serialize + // concurrent attempts to remove the same binding pymb_lock_registry(registry); - pymb_list_unlink(&binding->hook); + if (!binding->capsule) { + // Binding was concurrently removed from multiple places; the first + // one to get the registry lock wins. + pymb_unlock_registry(registry); + return; + } + +#if defined(Py_GIL_DISABLED) + // Determine if binding->pytype is still fully alive (not yet started + // finalizing). If so, it can't die until the next GC cycle, so freeing + // the binding at the next GC is safe. + PyObject* pytype_strong = NULL; + if (PyWeakref_GetRef(binding->pytype_wr, &pytype_strong) == -1) { + // If something's wrong with the weakref, leave pytype_strong set to + // NULL in order to conservatively assume the type is finalizing. + // This will leak the binding struct until the type object is destroyed. + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } +#endif + + // Clear the existing capsule's destructor so we don't have to worry about + // it firing after the pymb_binding struct has actually been freed. + // Note we can safely assume the capsule hasn't been freed yet, even + // though it might be mid-destruction. (Proof: Its destructor calls + // this function, which cannot complete until it acquires the lock we + // currently hold. If the destructor completed already, we would have bailed + // out above upon noticing capsule was already NULL.) + if (PyCapsule_SetDestructor(binding->capsule, NULL) != 0) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } + + // Mark this binding as being in the process of being destroyed. + binding->capsule = NULL; + + // If weakref hasn't fired yet, we don't need it anymore. Destroying it + // ensures it won't fire after the binding struct has been freed. + Py_CLEAR(binding->pytype_wr); + + pymb_list_unlink(&binding->link); + binding->framework->remove_local_binding(binding); PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { if (other != binding->framework) { other->remove_foreign_binding(binding); } } pymb_unlock_registry(registry); -} - -/* - * Increase the reference count of a binding. Return 1 if successful (you can - * use the binding and must call pymb_unref_binding() when done) or 0 if the - * binding is being removed and shouldn't be used. - */ -PYMB_FUNC int pymb_try_ref_binding(struct pymb_binding* binding) { -#if defined(Py_GIL_DISABLED) - if (!binding->framework->bindings_usable_forever) { -#if PY_VERSION_HEX >= 0x030e0000 - return PyUnstable_TryIncRef((PyObject *) binding->pytype); -#else - // bindings_usable_forever is required on this Python version, and - // was checked in pymb_add_framework() - assert(false); -#endif - } -#else - Py_INCREF((PyObject *) binding->pytype); -#endif - return 1; -} -/* Decrease the reference count of a binding. */ -PYMB_FUNC void pymb_unref_binding(struct pymb_binding* binding) { -#if defined(Py_GIL_DISABLED) - if (!binding->framework->bindings_usable_forever) { -#if PY_VERSION_HEX >= 0x030e0000 - Py_DECREF((PyObject *) binding->pytype); +#if !defined(Py_GIL_DISABLED) + // On GIL builds, there's no need to delay deallocation + binding->framework->free_local_binding(binding); #else - // bindings_usable_forever is required on this Python version, and - // was checked in pymb_add_framework() - assert(false); -#endif + // Create a new capsule to manage the actual freeing + PyObject* capsule_destroy = PyCapsule_New(binding, + "pymetabind_binding", + pymb_binding_capsule_destroy); + if (!capsule_destroy) { + // Just leak the binding if we can't set up the capsule + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } else if (pytype_strong) { + // Type still alive -> embed the capsule in a cycle so it lasts until + // next GC. (The type will live at least that long.) + PyObject* list = PyList_New(2); + if (!list) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + // leak the capsule and therefore the binding + } else { + PyList_SetItem(list, 0, capsule_destroy); + PyList_SetItem(list, 1, list); + // list is now referenced only by itself and will be GCable + } + } else { + // Type is dying -> destroy the capsule when the type is destroyed. + // Since the type's weakrefs were already cleared, any weakref we add + // now won't fire until the type's tp_dealloc. We reuse our existing + // weakref callback for convenience; the call that it makes to + // pymb_remove_binding() will be a no-op, but after it fires, + // the capsule destructor will do the freeing we desire. + PyObject* callback = PyCFunction_New(registry->weakref_callback_def, + capsule_destroy); + if (!callback) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + // leak the capsule and therefore the binding + } else { + Py_DECREF(capsule_destroy); // ownership transferred to callback + binding->pytype_wr = PyWeakref_NewRef((PyObject *) binding->pytype, + callback); + Py_DECREF(callback); // ownership transferred to weakref + if (!binding->pytype_wr) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } + } } -#else - Py_DECREF((PyObject *) binding->pytype); + Py_XDECREF(pytype_strong); #endif } From d0a02e2e26e34ce3db51c2319a3bc613269a9035 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 15 Sep 2025 23:11:00 -0600 Subject: [PATCH 10/18] Move pymetabind.h to include/pybind11/contrib/ --- .clang-tidy | 2 +- .pre-commit-config.yaml | 2 +- CMakeLists.txt | 2 +- include/pybind11/{detail => contrib}/pymetabind.h | 0 include/pybind11/detail/foreign.h | 3 ++- include/pybind11/detail/type_caster_base.h | 2 +- tests/extra_python_package/test_files.py | 7 +++++-- 7 files changed, 11 insertions(+), 7 deletions(-) rename include/pybind11/{detail => contrib}/pymetabind.h (100%) diff --git a/.clang-tidy b/.clang-tidy index 5285f80cad..200c3e3660 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -78,4 +78,4 @@ CheckOptions: value: true HeaderFilterRegex: 'pybind11/.*h' -ExcludeHeaderFilterRegex: 'pybind11/detail/pymetabind.h' +ExcludeHeaderFilterRegex: 'pybind11/contrib/.*h' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d85d399f5..eaedbf5b20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ ci: autoupdate_schedule: monthly # third-party content -exclude: ^(tools/JoinPaths.cmake|include/pybind11/detail/pymetabind.h)$ +exclude: ^(tools/JoinPaths.cmake|include/pybind11/contrib/.*)$ repos: diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e10b9198d..803f26bd45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,7 @@ if(PYBIND11_MASTER_PROJECT) endif() set(PYBIND11_HEADERS + include/pybind11/contrib/pymetabind.h include/pybind11/detail/class.h include/pybind11/detail/common.h include/pybind11/detail/cpp_conduit.h @@ -192,7 +193,6 @@ set(PYBIND11_HEADERS include/pybind11/detail/internals.h include/pybind11/detail/native_enum_data.h include/pybind11/detail/pybind11_namespace_macros.h - include/pybind11/detail/pymetabind.h include/pybind11/detail/struct_smart_holder.h include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h diff --git a/include/pybind11/detail/pymetabind.h b/include/pybind11/contrib/pymetabind.h similarity index 100% rename from include/pybind11/detail/pymetabind.h rename to include/pybind11/contrib/pymetabind.h diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index 1aa7373764..e01eca6d8a 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -9,9 +9,10 @@ #pragma once +#include + #include "common.h" #include "internals.h" -#include "pymetabind.h" #include "type_caster_base.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index e63559b4a9..f16f0b2ee4 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -12,13 +12,13 @@ #include #include #include +#include #include "common.h" #include "cpp_conduit.h" #include "descr.h" #include "dynamic_raw_ptr_cast_if_possible.h" #include "internals.h" -#include "pymetabind.h" #include "typeid.h" #include "using_smart_holder.h" #include "value_and_holder.h" diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 90e9fc5d6b..cd4512a873 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -87,7 +87,6 @@ "include/pybind11/detail/internals.h", "include/pybind11/detail/native_enum_data.h", "include/pybind11/detail/pybind11_namespace_macros.h", - "include/pybind11/detail/pymetabind.h", "include/pybind11/detail/struct_smart_holder.h", "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/typeid.h", @@ -96,6 +95,10 @@ "include/pybind11/detail/exception_translation.h", } +contrib_headers = { + "include/pybind11/contrib/pymetabind.h", +} + eigen_headers = { "include/pybind11/eigen/common.h", "include/pybind11/eigen/matrix.h", @@ -132,7 +135,7 @@ "share/pkgconfig/__init__.py", } -headers = main_headers | conduit_headers | detail_headers | eigen_headers | stl_headers +headers = main_headers | conduit_headers | detail_headers | contrib_headers | eigen_headers | stl_headers generated_files = cmake_files | pkgconfig_files all_files = headers | generated_files | py_files From 1958388512dcfbcd48a8ba9d7036c12baf964208 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 15 Sep 2025 23:41:42 -0600 Subject: [PATCH 11/18] Change some names containing 'foreign' to say 'interop' instead, so that we use 'foreign' specifically for types/frameworks that don't use our internals, rather than the concept of sharing with them --- include/pybind11/detail/class.h | 2 +- include/pybind11/detail/common.h | 2 +- include/pybind11/detail/foreign.h | 200 ++++++++++----------- include/pybind11/detail/internals.h | 32 ++-- include/pybind11/detail/type_caster_base.h | 50 +++--- include/pybind11/embed.h | 6 +- include/pybind11/pybind11.h | 8 +- include/pybind11/subinterpreter.h | 4 +- 8 files changed, 155 insertions(+), 149 deletions(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index e993e9e690..bd230ec6f2 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -226,7 +226,7 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { } else { internals.registered_types_cpp.erase(tindex); } - get_foreign_internals().copy_move_ctors.erase(tindex); + get_interop_internals().copy_move_ctors.erase(tindex); internals.registered_types_py.erase(tinfo->type); // Actually just `std::erase_if`, but that's only available in C++20 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 63140e8ffc..485994fdac 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -354,7 +354,7 @@ #define PYBIND11_ENSURE_INTERNALS_READY \ { \ pybind11::detail::get_internals_pp_manager().unref(); \ - pybind11::detail::get_foreign_internals_pp_manager().unref(); \ + pybind11::detail::get_interop_internals_pp_manager().unref(); \ pybind11::detail::get_internals(); \ } diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index e01eca6d8a..df378b59f6 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -20,8 +20,8 @@ PYBIND11_NAMESPACE_BEGIN(detail) // pybind11 exception translator that tries all known foreign ones PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { - auto &foreign_internals = get_foreign_internals(); - for (pymb_framework *fw : foreign_internals.exc_frameworks) { + auto &interop_internals = get_interop_internals(); + for (pymb_framework *fw : interop_internals.exc_frameworks) { try { fw->translate_exception(&p); } catch (...) { @@ -32,25 +32,25 @@ PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { } // When learning about a new foreign type, should we automatically use it? -inline bool should_autoimport_foreign(foreign_internals &foreign_internals, +inline bool should_autoimport_foreign(interop_internals &interop_internals, pymb_binding *binding) { - return foreign_internals.import_all && binding->framework->abi_lang == pymb_abi_lang_cpp - && binding->framework->abi_extra == foreign_internals.self->abi_extra; + return interop_internals.import_all && binding->framework->abi_lang == pymb_abi_lang_cpp + && binding->framework->abi_extra == interop_internals.self->abi_extra; } // Add the given `binding` to our type maps so that we can use it to satisfy // from- and to-Python requests for the given C++ type inline void import_foreign_binding(pymb_binding *binding, const std::type_info *cpptype) noexcept { // Caller must hold the internals lock - auto &foreign_internals = get_foreign_internals(); - foreign_internals.imported_any = true; - foreign_internals.bindings.emplace(*cpptype, binding); + auto &interop_internals = get_interop_internals(); + interop_internals.imported_any = true; + interop_internals.bindings.emplace(*cpptype, binding); } // Callback functions for other frameworks to operate on our objects // or tell us about theirs -inline void *foreign_cb_from_python(pymb_binding *binding, +inline void *interop_cb_from_python(pymb_binding *binding, PyObject *pyobj, uint8_t convert, void (*keep_referenced)(void *ctx, PyObject *obj), @@ -101,7 +101,7 @@ inline void *foreign_cb_from_python(pymb_binding *binding, return ret; } -inline PyObject *foreign_cb_to_python(pymb_binding *binding, +inline PyObject *interop_cb_to_python(pymb_binding *binding, void *cobj, enum pymb_rv_policy rvp_, PyObject *parent) noexcept { @@ -119,9 +119,9 @@ inline PyObject *foreign_cb_to_python(pymb_binding *binding, copy_or_move_ctor copy_ctor = nullptr, move_ctor = nullptr; if (rvp == return_value_policy::copy || rvp == return_value_policy::move) { with_internals([&](internals &) { - auto &foreign_internals = get_foreign_internals(); - auto it = foreign_internals.copy_move_ctors.find(*ti->cpptype); - if (it != foreign_internals.copy_move_ctors.end()) { + auto &interop_internals = get_interop_internals(); + auto it = interop_internals.copy_move_ctors.find(*ti->cpptype); + if (it != interop_internals.copy_move_ctors.end()) { std::tie(copy_ctor, move_ctor) = it->second; } }); @@ -135,7 +135,7 @@ inline PyObject *foreign_cb_to_python(pymb_binding *binding, } } -inline int foreign_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void *)) noexcept { +inline int interop_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void *)) noexcept { try { if (!cb) { keep_alive_impl(nurse, static_cast(payload)); @@ -150,7 +150,7 @@ inline int foreign_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void } } -inline void foreign_cb_translate_exception(const void *eptr) { +inline void interop_cb_translate_exception(const void *eptr) { with_exception_translators( [&](std::forward_list &exception_translators, std::forward_list &local_exception_translators) { @@ -179,7 +179,7 @@ inline void foreign_cb_translate_exception(const void *eptr) { // frameworks' exceptions), it's the second-last one and should // be skipped too. We don't want mutual recursion between // different frameworks' translators. - if (!get_foreign_internals().exc_frameworks.empty()) { + if (!get_interop_internals().exc_frameworks.empty()) { ++leader; } @@ -212,33 +212,33 @@ inline void foreign_cb_translate_exception(const void *eptr) { }); } -inline void foreign_cb_add_foreign_binding(pymb_binding *binding) noexcept { +inline void interop_cb_remove_local_binding(pymb_binding *binding) noexcept { with_internals([&](internals &) { - auto &foreign_internals = get_foreign_internals(); - if (should_autoimport_foreign(foreign_internals, binding)) { + auto &interop_internals = get_interop_internals(); + if (should_autoimport_foreign(interop_internals, binding)) { import_foreign_binding(binding, (const std::type_info *) binding->native_type); } }); } -inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { +inline void interop_cb_remove_foreign_binding(pymb_binding *binding) noexcept { with_internals([&](internals &) { - auto &foreign_internals = get_foreign_internals(); + auto &interop_internals = get_interop_internals(); auto remove_from_type = [&](const std::type_info *type) { - auto range = foreign_internals.bindings.equal_range(*type); + auto range = interop_internals.bindings.equal_range(*type); for (auto it = range.first; it != range.second; ++it) { if (it->second == binding) { - foreign_internals.bindings.erase(it); + interop_internals.bindings.erase(it); break; } } }; - bool should_remove_auto = should_autoimport_foreign(foreign_internals, binding); - auto it = foreign_internals.manual_imports.find(binding); - if (it != foreign_internals.manual_imports.end()) { + bool should_remove_auto = should_autoimport_foreign(interop_internals, binding); + auto it = interop_internals.manual_imports.find(binding); + if (it != interop_internals.manual_imports.end()) { remove_from_type(it->second); should_remove_auto &= (it->second != binding->native_type); - foreign_internals.manual_imports.erase(it); + interop_internals.manual_imports.erase(it); } if (should_remove_auto) { remove_from_type((const std::type_info *) binding->native_type); @@ -246,13 +246,13 @@ inline void foreign_cb_remove_foreign_binding(pymb_binding *binding) noexcept { }); } -inline void foreign_cb_add_foreign_framework(pymb_framework *framework) noexcept { +inline void interop_cb_add_foreign_framework(pymb_framework *framework) noexcept { if (framework->translate_exception) { with_exception_translators( [&](std::forward_list &exception_translators, std::forward_list &) { - auto &foreign_internals = get_foreign_internals(); - if (foreign_internals.exc_frameworks.empty()) { + auto &interop_internals = get_interop_internals(); + if (interop_internals.exc_frameworks.empty()) { // First foreign framework with an exception translator. // Add our `foreign_exception_translator` wrapper in the // 2nd-last position (last is the default exception @@ -265,12 +265,12 @@ inline void foreign_cb_add_foreign_framework(pymb_framework *framework) noexcept exception_translators.insert_after(trailer, foreign_exception_translator); } // Add the new framework at the end of the list - auto leader = foreign_internals.exc_frameworks.begin(); + auto leader = interop_internals.exc_frameworks.begin(); auto trailer = leader; - while (++leader != foreign_internals.exc_frameworks.end()) { + while (++leader != interop_internals.exc_frameworks.end()) { ++trailer; } - foreign_internals.exc_frameworks.insert_after(trailer, framework); + interop_internals.exc_frameworks.insert_after(trailer, framework); }); } } @@ -278,7 +278,7 @@ inline void foreign_cb_add_foreign_framework(pymb_framework *framework) noexcept // (end of callbacks) // Advertise our existence, and the above callbacks, to other frameworks -PYBIND11_NOINLINE bool foreign_internals::initialize() { +PYBIND11_NOINLINE bool interop_internals::initialize() { bool inited_by_us = with_internals([&](internals &) { if (registry) { return false; @@ -294,57 +294,57 @@ PYBIND11_NOINLINE bool foreign_internals::initialize() { self->leak_safe = 0; self->abi_lang = pymb_abi_lang_cpp; self->abi_extra = PYBIND11_PLATFORM_ABI_ID; - self->from_python = foreign_cb_from_python; - self->to_python = foreign_cb_to_python; - self->keep_alive = foreign_cb_keep_alive; - self->translate_exception = foreign_cb_translate_exception; - self->add_foreign_binding = foreign_cb_add_foreign_binding; - self->remove_foreign_binding = foreign_cb_remove_foreign_binding; - self->add_foreign_framework = foreign_cb_add_foreign_framework; + self->from_python = interop_cb_from_python; + self->to_python = interop_cb_to_python; + self->keep_alive = interop_cb_keep_alive; + self->translate_exception = interop_cb_translate_exception; + self->add_foreign_binding = interop_cb_add_foreign_binding; + self->remove_foreign_binding = interop_cb_remove_foreign_binding; + self->add_foreign_framework = interop_cb_add_foreign_framework; return true; }); if (inited_by_us) { // Unlock internals before calling add_framework, so that the callbacks - // (foreign_cb_add_foreign_binding, etc) can safely re-lock it. + // (interop_cb_add_foreign_binding, etc) can safely re-lock it. pymb_add_framework(registry, self.get()); } return inited_by_us; } -inline foreign_internals::~foreign_internals() = default; +inline interop_internals::~interop_internals() = default; // Learn to satisfy from- and to-Python requests for `cpptype` using the // foreign binding provided by the given `pytype`. If cpptype is nullptr, infer // the C++ type by looking at the binding, and require that its ABI match ours. // Throws an exception on failure. Caller must hold the internals lock and have -// already called foreign_internals.initialize_if_needed(). -PYBIND11_NOINLINE void import_foreign_type(type pytype, const std::type_info *cpptype) { - auto &foreign_internals = get_foreign_internals(); +// already called interop_internals.initialize_if_needed(). +PYBIND11_NOINLINE void import_for_interop(handle pytype, const std::type_info *cpptype) { + auto &interop_internals = get_interop_internals(); pymb_binding *binding = pymb_get_binding(pytype.ptr()); if (!binding) { - pybind11_fail("pybind11::import_foreign_type(): type does not define " + pybind11_fail("pybind11::import_for_interop(): type does not define " "a __pymetabind_binding__"); } - if (binding->framework == foreign_internals.self.get()) { - pybind11_fail("pybind11::import_foreign_type(): type is not foreign"); + if (binding->framework == interop_internals.self.get()) { + pybind11_fail("pybind11::import_for_interop(): type is not foreign"); } if (!cpptype) { if (binding->framework->abi_lang != pymb_abi_lang_cpp) { - pybind11_fail("pybind11::import_foreign_type(): type is not " + pybind11_fail("pybind11::import_for_interop(): type is not " "written in C++, so you must specify a C++ type"); } - if (binding->framework->abi_extra != foreign_internals.self->abi_extra) { - pybind11_fail("pybind11::import_foreign_type(): type has " + if (binding->framework->abi_extra != interop_internals.self->abi_extra) { + pybind11_fail("pybind11::import_for_interop(): type has " "incompatible C++ ABI with this module"); } cpptype = (const std::type_info *) binding->native_type; } - auto result = foreign_internals.manual_imports.emplace(binding, cpptype); + auto result = interop_internals.manual_imports.emplace(binding, cpptype); if (!result.second) { const auto *existing = (const std::type_info *) result.first->second; if (existing != cpptype && *existing != *cpptype) { - pybind11_fail("pybind11::import_foreign_type(): type was " + pybind11_fail("pybind11::import_for_interop(): type was " "already imported as a different C++ type"); } } @@ -354,54 +354,54 @@ PYBIND11_NOINLINE void import_foreign_type(type pytype, const std::type_info *cp // Call `import_foreign_binding()` for every ABI-compatible type provided by // other C++ binding frameworks used by extension modules loaded in this // interpreter, both those that exist now and those bound in the future. -PYBIND11_NOINLINE void foreign_enable_import_all() { - auto &foreign_internals = get_foreign_internals(); +PYBIND11_NOINLINE void interop_enable_import_all() { + auto &interop_internals = get_interop_internals(); bool proceed = with_internals([&](internals &) { - if (foreign_internals.import_all) { + if (interop_internals.import_all) { return false; } - foreign_internals.import_all = true; + interop_internals.import_all = true; return true; }); if (!proceed) { return; } - if (foreign_internals.initialize_if_needed()) { + if (interop_internals.initialize_if_needed()) { // pymb_add_framework tells us about every existing type when we // register, so if we register with import enabled, we're done return; } // If we enable import after registering, we have to iterate over the // list of types ourselves. Do this without the internals lock held so - // we can reuse the pymb callback functions. foreign_internals registry + + // we can reuse the pymb callback functions. interop_internals registry + // self never change once they're non-null, so we can access them // without locking here. - pymb_lock_registry(foreign_internals.registry); + pymb_lock_registry(interop_internals.registry); // NOLINTNEXTLINE(modernize-use-auto) - PYMB_LIST_FOREACH(struct pymb_binding *, binding, foreign_internals.registry->bindings) { - if (binding->framework != foreign_internals.self.get() + PYMB_LIST_FOREACH(struct pymb_binding *, binding, interop_internals.registry->bindings) { + if (binding->framework != interop_internals.self.get() && pymb_try_ref_binding(binding) != 0) { - foreign_cb_add_foreign_binding(binding); + interop_cb_add_foreign_binding(binding); pymb_unref_binding(binding); } } - pymb_unlock_registry(foreign_internals.registry); + pymb_unlock_registry(interop_internals.registry); } // Expose hooks for other frameworks to use to work with the given pybind11 -// type object. Caller must hold the internals lock and have already called -// foreign_internals.initialize_if_needed(). -PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { - auto &foreign_internals = get_foreign_internals(); - auto range = foreign_internals.bindings.equal_range(*ti->cpptype); +// type object. Caller must hold the internals lock and have already called +// interop_internals.initialize_if_needed(). +PYBIND11_NOINLINE void export_for_interop(type_info *ti) { + auto &interop_internals = get_interop_internals(); + auto range = interop_internals.bindings.equal_range(*ti->cpptype); for (auto it = range.first; it != range.second; ++it) { - if (it->second->framework == foreign_internals.self.get()) { + if (it->second->framework == interop_internals.self.get()) { return; // already exported } } auto *binding = new pymb_binding{}; - binding->framework = foreign_internals.self.get(); + binding->framework = interop_internals.self.get(); binding->pytype = ti->type; binding->native_type = ti->cpptype; binding->source_name = PYBIND11_COMPAT_STRDUP(clean_type_id(ti->cpptype->name()).c_str()); @@ -409,35 +409,35 @@ PYBIND11_NOINLINE void export_type_to_foreign(type_info *ti) { capsule tie_lifetimes((void *) binding, [](void *p) { auto *binding = (pymb_binding *) p; - pymb_remove_binding(get_foreign_internals().registry, binding); + pymb_remove_binding(get_interop_internals().registry, binding); free(const_cast(binding->source_name)); delete binding; }); keep_alive_impl((PyObject *) ti->type, tie_lifetimes); - foreign_internals.bindings.emplace(*ti->cpptype, binding); - pymb_add_binding(foreign_internals.registry, binding); + interop_internals.bindings.emplace(*ti->cpptype, binding); + pymb_add_binding(interop_internals.registry, binding); } // Call `export_type_to_foreign()` for each type that currently exists in our // internals structure and each type created in the future. -PYBIND11_NOINLINE void foreign_enable_export_all() { - auto &foreign_internals = get_foreign_internals(); +PYBIND11_NOINLINE void interop_enable_export_all() { + auto &interop_internals = get_interop_internals(); bool proceed = with_internals([&](internals &) { - if (foreign_internals.export_all) { + if (interop_internals.export_all) { return false; } - foreign_internals.export_all = true; - foreign_internals.export_type_to_foreign = &detail::export_type_to_foreign; + interop_internals.export_all = true; + interop_internals.export_for_interop = &detail::export_for_interop; return true; }); if (!proceed) { return; } - foreign_internals.initialize_if_needed(); + interop_internals.initialize_if_needed(); with_internals([&](internals &internals) { for (const auto &entry : internals.registered_types_cpp) { - detail::export_type_to_foreign(entry.second); + detail::export_for_interop(entry.second); } }); } @@ -450,11 +450,11 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, pymb_binding *binding), void *closure) { auto &internals = get_internals(); - auto &foreign_internals = get_foreign_internals(); + auto &interop_internals = get_interop_internals(); PYBIND11_LOCK_INTERNALS(internals); (void) internals; // suppress unused warning on non-ft builds - auto range = foreign_internals.bindings.equal_range(*type); + auto range = interop_internals.bindings.equal_range(*type); if (range.first == range.second) { return nullptr; // no foreign bindings @@ -481,7 +481,7 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, #ifndef Py_GIL_DISABLED for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() + if (binding->framework != interop_internals.self.get() && pymb_try_ref_binding(binding) != 0) { void *result = attempt(closure, binding); pymb_unref_binding(binding); @@ -510,7 +510,7 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, // our scratch storage for (auto it = range.first; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework != foreign_internals.self.get() + if (binding->framework != interop_internals.self.get() && pymb_try_ref_binding(binding) != 0) { *scratch_tail++ = binding; } @@ -534,34 +534,34 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, PYBIND11_NAMESPACE_END(detail) -inline void set_foreign_type_defaults(bool export_all, bool import_all) { - auto &foreign_internals = detail::get_foreign_internals(); - if (import_all && !foreign_internals.import_all) { - detail::foreign_enable_import_all(); +inline void interoperate_by_default(bool export_all = true, bool import_all = true) { + auto &interop_internals = detail::get_interop_internals(); + if (import_all && !interop_internals.import_all) { + detail::interop_enable_import_all(); } - if (export_all && !foreign_internals.export_all) { - detail::foreign_enable_export_all(); + if (export_all && !interop_internals.export_all) { + detail::interop_enable_export_all(); } } template -inline void import_foreign_type(type pytype) { +inline void import_for_interop(type pytype) { const std::type_info *cpptype = std::is_void::value ? nullptr : &typeid(T); - auto &foreign_internals = detail::get_foreign_internals(); - foreign_internals.initialize_if_needed(); + auto &interop_internals = detail::get_interop_internals(); + interop_internals.initialize_if_needed(); detail::with_internals( - [&](detail::internals &) { detail::import_foreign_type(std::move(pytype), cpptype); }); + [&](detail::internals &) { detail::import_for_interop(std::move(pytype), cpptype); }); } -inline void export_type_to_foreign(type ty) { +inline void export_for_interop(type ty) { + auto &interop_internals = detail::get_interop_internals(); + interop_internals.initialize_if_needed(); detail::type_info *ti = detail::get_type_info((PyTypeObject *) ty.ptr()); if (!ti) { pybind11_fail("pybind11::export_type_to_foreign: not a " "pybind11 registered type"); } - auto &foreign_internals = detail::get_foreign_internals(); - foreign_internals.initialize_if_needed(); - detail::with_internals([&](detail::internals &) { detail::export_type_to_foreign(ti); }); + detail::with_internals([&](detail::internals &) { detail::export_for_interop(ti); }); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 200533a2c3..e0bab997a0 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -311,7 +311,7 @@ using copy_or_move_ctor = void *(*) (const void *); // version, it would be good for performance to also add a flag to `type_info` // indicating whether any foreign bindings are also known for its C++ type; // that way we can avoid an extra lookup when conversion to a native type fails. -struct foreign_internals { +struct interop_internals { // Registered foreign bindings for each C++ type. // Protected by internals::mutex. type_multimap bindings; @@ -337,26 +337,26 @@ struct foreign_internals { std::unique_ptr self; // Remember the C++ type associated with each binding by - // import_type_from_foreign(), so we can clean up `bindings` properly. + // import_for_interop(), so we can clean up `bindings` properly. // Protected by internals::mutex. std::unordered_map manual_imports; - // Pointer to `detail::export_type_to_foreign` in foreign.h, or nullptr if + // Pointer to `detail::export_for_interop` in foreign.h, or nullptr if // export_all is false. This indirection is vital to avoid having every // compilation unit with a py::class_ pull in the callback methods in // foreign.h. Instead, only compilation units that call - // set_foreign_type_defaults(), import_foreign_type(), or - // export_type_to_foreign() will emit that code. - void (*export_type_to_foreign)(type_info *); + // interoperate_by_default(), import_for_interop(), or + // export_for_interop() will emit that code. + void (*export_for_interop)(type_info *); // Should we automatically advertise our types to other binding frameworks, - // or only when requested via pybind11::export_type_to_foreign()? + // or only when requested via pybind11::export_for_interop()? // Never becomes false once it is set to true. bool export_all = false; // Should we automatically use types advertised by other frameworks as // a fallback when we can't do a cast using pybind11 types, or only when - // requested via pybind11::import_foreign_type()? + // requested via pybind11::import_for_interop()? // Never becomes false once it is set to true. bool import_all = false; @@ -364,7 +364,7 @@ struct foreign_internals { // own types? bool imported_any = false; - inline ~foreign_internals(); + inline ~interop_internals(); // Returns true if we initialized, false if someone else already did. inline bool initialize_if_needed() { @@ -778,17 +778,17 @@ inline auto with_exception_translators(const F &cb) local_internals.registered_exception_translators); } -inline internals_pp_manager &get_foreign_internals_pp_manager() { - static internals_pp_manager foreign_internals_pp_manager( - PYBIND11_INTERNALS_ID "foreign", nullptr); - return foreign_internals_pp_manager; +inline internals_pp_manager &get_interop_internals_pp_manager() { + static internals_pp_manager interop_internals_pp_manager( + PYBIND11_INTERNALS_ID "interop", nullptr); + return interop_internals_pp_manager; } -inline foreign_internals &get_foreign_internals() { - auto &ppmgr = get_foreign_internals_pp_manager(); +inline interop_internals &get_interop_internals() { + auto &ppmgr = get_interop_internals_pp_manager(); auto &ptr = *ppmgr.get_pp(); if (!ptr) { - ptr.reset(new foreign_internals()); + ptr.reset(new interop_internals()); } return *ptr; } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index f16f0b2ee4..8cfbfee33d 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -265,10 +265,10 @@ PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, return handle((PyObject *) type_info->type); } if (foreign_ok) { - auto &foreign_internals = detail::get_foreign_internals(); - if (foreign_internals.imported_any) { + auto &interop_internals = detail::get_interop_internals(); + if (interop_internals.imported_any) { handle ret = with_internals([&](internals &) { - auto range = foreign_internals.bindings.equal_range(tp); + auto range = interop_internals.bindings.equal_range(tp); if (range.first != range.second) { return handle((PyObject *) range.first->second->pytype); } @@ -1052,7 +1052,7 @@ class type_caster_generic { if (!srcs.result.second) { // No pybind11 type info. See if we can use another framework's // type to complete this cast. Set srcs.used_foreign if so. - if (get_foreign_internals().imported_any) { + if (get_interop_internals().imported_any) { if (handle ret = cast_foreign(srcs, policy, parent)) { return ret; } @@ -1215,10 +1215,11 @@ class type_caster_generic { return nullptr; } - /// Try to load as a type exposed by a different binding framework. + /// Try to load as a type exposed by a different binding framework (which + /// might be an ABI-incompatible version of pybind11). bool try_load_other_framework(handle src, bool convert) { - auto &foreign_internals = get_foreign_internals(); - if (!foreign_internals.imported_any || !cpptype || src.is_none()) { + auto &interop_internals = get_interop_internals(); + if (!interop_internals.imported_any || !cpptype || src.is_none()) { return false; } @@ -1245,30 +1246,35 @@ class type_caster_generic { return false; } - /// Try to load with foreign typeinfo, if available. Used when there is no - /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign(handle src, bool convert) { - constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; - const auto pytype = type::handle_of(src); - if (!hasattr(pytype, local_key)) { - return try_load_other_framework(src, convert); - } - - type_info *foreign_typeinfo = reinterpret_borrow(getattr(pytype, local_key)); - // Only consider this foreign loader if actually foreign and is a loader of the correct cpp - // type - if (foreign_typeinfo->module_local_load == &local_load - || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) { + /// Try to load as a type bound as py::module_local() in a different (but + /// ABI-compatible) pybind11 module. + bool try_load_other_module_local(handle src, type_info *remote_typeinfo) { + // Only consider this loader if it's not ours and it loads the correct cpp type + if (remote_typeinfo->module_local_load == &local_load + || (cpptype && !same_type(*cpptype, *remote_typeinfo->cpptype))) { return false; } - if (auto *result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { + if (auto *result = remote_typeinfo->module_local_load(src.ptr(), remote_typeinfo)) { value = result; return true; } return false; } + /// Try to load with foreign typeinfo, if available. Used when there is no + /// native typeinfo, or when the native one wasn't able to produce a value. + PYBIND11_NOINLINE bool try_load_foreign(handle src, bool convert) { + constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; + const auto pytype = type::handle_of(src); + if (hasattr(pytype, local_key)) { + return try_load_other_module_local( + src, reinterpret_borrow(getattr(pytype, local_key))); + } else { + return try_load_other_framework(src, convert); + } + } + // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant // bits of code between here and copyable_holder_caster where the two classes need different // logic (without having to resort to virtual inheritance). diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 830dccb10b..47d7e33aa2 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -245,7 +245,7 @@ inline void finalize_interpreter() { if (detail::get_num_interpreters_seen() > 1) { detail::get_internals_pp_manager().unref(); detail::get_local_internals_pp_manager().unref(); - detail::get_foreign_internals_pp_manager().unref(); + detail::get_interop_internals_pp_manager().unref(); // We know there can be no other interpreter alive now, so we can lower the count detail::get_num_interpreters_seen() = 1; @@ -257,7 +257,7 @@ inline void finalize_interpreter() { // and check it after Py_Finalize(). detail::get_internals_pp_manager().get_pp(); detail::get_local_internals_pp_manager().get_pp(); - detail::get_foreign_internals_pp_manager().get_pp(); + detail::get_interop_internals_pp_manager().get_pp(); Py_Finalize(); @@ -266,7 +266,7 @@ inline void finalize_interpreter() { // interpreter detail::get_internals_pp_manager().destroy(); detail::get_local_internals_pp_manager().destroy(); - detail::get_foreign_internals_pp_manager().destroy(); + detail::get_interop_internals_pp_manager().destroy(); // We know there is no interpreter alive now, so we can reset the count detail::get_num_interpreters_seen() = 0; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 37bce456d5..351b1b209f 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1648,9 +1648,9 @@ class generic_type : public object { internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; PYBIND11_WARNING_POP - auto &foreign_internals = get_foreign_internals(); - if (foreign_internals.export_all) { - foreign_internals.export_type_to_foreign(tinfo); + auto &interop_internals = get_interop_internals(); + if (interop_internals.export_all) { + interop_internals.export_for_interop(tinfo); } }); @@ -2136,7 +2136,7 @@ class class_ : public detail::generic_type { generic_type::initialize(record); with_internals([&](internals &internals) { - get_foreign_internals().copy_move_ctors.emplace( + get_interop_internals().copy_move_ctors.emplace( *record.type, detail::type_caster_base::copy_and_move_ctors()); if (has_alias) { auto &instances = record.module_local ? get_local_internals().registered_types_cpp diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index f0b8cb367b..de4cd4a5ee 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -180,7 +180,7 @@ class subinterpreter { // internals themselves. detail::get_internals_pp_manager().get_pp(); detail::get_local_internals_pp_manager().get_pp(); - detail::get_foreign_internals_pp_manager().get_pp(); + detail::get_interop_internals_pp_manager().get_pp(); // End it Py_EndInterpreter(destroy_tstate); @@ -189,7 +189,7 @@ class subinterpreter { // py::capsule calls `get_internals()` during destruction), so we destroy afterward. detail::get_internals_pp_manager().destroy(); detail::get_local_internals_pp_manager().destroy(); - detail::get_foreign_internals_pp_manager().destroy(); + detail::get_interop_internals_pp_manager().destroy(); // switch back to the old tstate and old GIL (if there was one) if (switch_back) From 18e8808a7bdea04d87ed3a328f2abe2c6284ca4e Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 15 Sep 2025 23:44:58 -0600 Subject: [PATCH 12/18] Add a __pybind11_enum__ capsule to native enums containing some information needed for interop with them, plus a destructor that unregisters the enum when it's destroyed --- include/pybind11/detail/internals.h | 9 ++++++ include/pybind11/detail/native_enum_data.h | 33 ++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index e0bab997a0..79b2172142 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -431,6 +431,15 @@ struct type_info { bool module_local : 1; }; +/// Information stored in a capsule on py::native_enum() types. +struct native_enum_info { + const std::type_info *cpptype; + uint32_t size_bytes; + bool is_signed; + + static const char *attribute_name() { return "__pybind11_enum__"; } +}; + #define PYBIND11_ABI_TAG \ "v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index a8f7675ba0..fe4ec82778 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -24,15 +24,25 @@ native_enum_missing_finalize_error_message(const std::string &enum_name_encoded) class native_enum_data { public: - native_enum_data(const object &parent_scope, + native_enum_data(handle parent_scope, const char *enum_name, const char *native_type_name, const char *class_doc, - const std::type_index &enum_type_index) + const native_enum_info &enum_info_) : enum_name_encoded{enum_name}, native_type_name_encoded{native_type_name}, - enum_type_index{enum_type_index}, parent_scope(parent_scope), enum_name{enum_name}, + enum_type_index{*enum_info_.cpptype}, parent_scope(parent_scope), enum_name{enum_name}, native_type_name{native_type_name}, class_doc(class_doc), export_values_flag{false}, - finalize_needed{false} {} + finalize_needed{false} { + enum_info = capsule(new native_enum_info{enum_info_}, + native_enum_info::attribute_name(), + +[](void *enum_info_) { + auto *info = (native_enum_info *) enum_info_; + with_internals([&](internals &internals) { + internals.native_enum_type_map.erase(*info->cpptype); + }); + delete info; + }); + } void finalize(); @@ -67,10 +77,11 @@ class native_enum_data { std::type_index enum_type_index; private: - object parent_scope; + handle parent_scope; str enum_name; str native_type_name; std::string class_doc; + capsule enum_info; protected: list members; @@ -81,12 +92,6 @@ class native_enum_data { bool finalize_needed : 1; }; -inline void global_internals_native_enum_type_map_set_item(const std::type_index &enum_type_index, - PyObject *py_enum) { - with_internals( - [&](internals &internals) { internals.native_enum_type_map[enum_type_index] = py_enum; }); -} - inline handle global_internals_native_enum_type_map_get_item(const std::type_index &enum_type_index) { return with_internals([&](internals &internals) { @@ -202,7 +207,11 @@ inline void native_enum_data::finalize() { for (auto doc : member_docs) { py_enum[doc[int_(0)]].attr("__doc__") = doc[int_(1)]; } - global_internals_native_enum_type_map_set_item(enum_type_index, py_enum.release().ptr()); + + py_enum.attr(native_enum_info::attribute_name()) = enum_info; + with_internals([&](internals &internals) { + internals.native_enum_type_map[enum_type_index] = py_enum.ptr(); + }); } PYBIND11_NAMESPACE_END(detail) From 647df8bc7977dbbcfa0cc54e0835b91b5d081af1 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 15 Sep 2025 23:45:37 -0600 Subject: [PATCH 13/18] Fix all known bugs with interop support, update to pymetabind 0.3, add docs and tests --- docs/advanced/classes.rst | 11 +- docs/advanced/interop.rst | 296 +++++++++++ docs/index.rst | 1 + docs/upgrade.rst | 49 ++ include/pybind11/cast.h | 43 +- include/pybind11/contrib/pymetabind.h | 123 +++-- include/pybind11/detail/common.h | 27 + include/pybind11/detail/foreign.h | 528 +++++++++++++------ include/pybind11/detail/internals.h | 19 +- include/pybind11/detail/native_enum_data.h | 7 + include/pybind11/detail/type_caster_base.h | 159 ++++-- include/pybind11/native_enum.h | 13 +- include/pybind11/pybind11.h | 21 +- tests/CMakeLists.txt | 2 + tests/test_interop.h | 97 ++++ tests/test_interop.py | 564 +++++++++++++++++++++ tests/test_interop_1.cpp | 35 ++ tests/test_interop_2.cpp | 37 ++ tests/test_interop_3.cpp | 229 +++++++++ 19 files changed, 1979 insertions(+), 282 deletions(-) create mode 100644 docs/advanced/interop.rst create mode 100644 tests/test_interop.h create mode 100644 tests/test_interop.py create mode 100644 tests/test_interop_1.cpp create mode 100644 tests/test_interop_2.cpp create mode 100644 tests/test_interop_3.cpp diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 14bfc0bcdb..faaba38b8d 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -972,9 +972,14 @@ Module-local class bindings =========================== When creating a binding for a class, pybind11 by default makes that binding -"global" across modules. What this means is that a type defined in one module -can be returned from any module resulting in the same Python type. For -example, this allows the following: +"global" across modules. What this means is that instances whose type is +defined with a ``py::class_`` statement in one module can be passed to or +returned from a function defined in any other module that is "ABI compatible" +with the first, i.e., that was built with sufficiently similar versions of +pybind11 and of the C++ compiler and C++ standard library. The internal data +structures that pybind11 uses to keep track of its types and instances are +shared just as they would be if everything were in the same module. +For example, this allows the following: .. code-block:: cpp diff --git a/docs/advanced/interop.rst b/docs/advanced/interop.rst new file mode 100644 index 0000000000..64e8c06799 --- /dev/null +++ b/docs/advanced/interop.rst @@ -0,0 +1,296 @@ +.. _interop: + +Interoperating with foreign bindings +==================================== + +When you bind a function with pybind11 that has a parameter of type ``T``, +its typical behavior (if ``T`` does not use a :ref:`built-in ` +or :ref:`custom type caster `) is to only accept arguments +for that parameter that are Python instances of the type created by a +``py::class_(...)`` binding statement, or that derive from that type, +or that match a defined :ref:`implicit conversion ` +to that type (``py::implicitly_convertible()``). Moreover, +if the ``py::class_(...)`` binding statement was written in a different +pybind11 extension than the function that needs the ``T``, the two extensions +must be ABI-compatible: they must use similar enough versions of pybind11 that +it's safe for their respective copies of pybind11 to share their data +structures with each other. + +Sometimes, you might want more flexibility than that: + +- Perhaps you have a large codebase containing a number of different pybind11 + extension modules that share types with each other, and you want to upgrade + to a new and ABI-incompatible release of pybind11 in some fashion other than + "upgrade every module at the same time". + +- Perhaps you need to work with types provided by a third-party extension + such as PyTorch, which uses pybind11 but not the version you prefer. + +- Perhaps you'd like to port some of the especially performance-sensitive + parts of your bindings to a faster but less featureful binding framework, + without leaving the comfortable world of pybind11 behind entirely. + +To handle such situations, pybind11 can be taught to interoperate with bindings +that were not created using pybind11, or that were created with an +ABI-incompatible version of pybind11 (as long as it is new enough to support +this feature). For example, you can define a class binding for ``Pet`` in one +extension that is written using pybind11, and then write function bindings for +``void groom(Pet&)`` and ``Pet clone(const Pet&)`` in a separate extension +module that is written using `nanobind `__, or +vice versa. The interoperability mechanism described here allows each framework +to figure out (among other things) how to get a reference to a C++ ``Pet`` out +of a Python object provided by the other framework that supposedly contains a +Pet, without knowing anything about how that framework lays out its instances. +From pybind11's perspective, nanobind and its bindings are considered "foreign". + +In order for pybind11 to interoperate with another framework in this way, the +other framework must support the `pymetabind +`__ standard. See that link for +a list of frameworks that claim to do so. + +Exporting pybind11 bindings for other frameworks to use +------------------------------------------------------- + +In order for a type bound by pybind11 to be usable by other binding frameworks, +pybind11 must allocate a small data structure describing how others should work +with that type. While the overhead of this is low, it is not zero, so pybind11 +only does so for types where you request it. Pass the Python type object to +``py::export_for_interop()``, or use ``py::interoperate_by_default()`` if you +want all types to be exported automatically as soon as they are bound. + +You can use ``py::type::of()`` to get the Python type object for +a C++ type. For example: + +.. code-block:: cpp + + PYBIND11_MODULE(my_ext, m) { + auto pet = py::class_(m, "Pet") + .def(py::init()) + .def("speak", &Pet::speak); + + // These two lines are equivalent: + py::export_for_interop(pet); + py::export_for_interop(py::type::of()); + } + + +Importing other frameworks' bindings for pybind11 to use +-------------------------------------------------------- + +In order for pybind11 to interoperate with a foreign type, the foreign framework +that bound the type must have created an interoperability record for it. +Depending on the framework, this might occur automatically or might require +an operation similar to the ``py::export_for_interop()`` described in the +previous section. (You can tell if this has happened by checking for the +presence of an attribute on the type object called ``__pymetabind_binding__``.) +Consult the other framework's documentation for details. + +Once that's done, you can teach pybind11 about the foreign type by passing its +Python type object to ``py::import_for_interop()``. +This function takes an optional template argument specifying which C++ type to +associate the Python type with. If the foreign type was bound using another +C++ framework, such as nanobind or a different version of pybind11, the template +argument need not be provided because the C++ ``std::type_info`` structure +describing the type can be found by looking at the interoperability record. +On the other hand, if the foreign type is not written in C++ or is bound by +a non-C++ framework that doesn't know about ``std::type_info``, pybind11 won't +be able to figure out what the C++ type is, and needs you to specify it via +a template argument to ``py::import_for_inteorp()``. + +If you *don't* supply a template argument (for importing a C++ type), then +pybind11 will check for you that the binding you're adding was compiled using a +platform C++ ABI that is consistent with the build options for your pybind11 +extension. This helps to ensure that the exporter and importer mean the same +thing when they say, for example, ``std::vector``. +The import will throw an exception if an incompatibility is detected. + +If you *do* supply a template argument (for importing a +different-language type and specifying the C++ equivalent), pybind11 +will assume that you have validated compatibility yourself. Getting it +wrong can cause crashes and other sorts of undefined behavior, so if +you're working with bindings that were created in another language, make +doubly sure you're specifying a C++ type that is fully ABI-compatible with +the one used by the foreign binding. + +You can use ``py::interoperate_by_default()`` if you want pybind11 to +automatically import every compatible C++ type as soon as it has been +exported by another framework. + +.. code-block:: cpp + + // --- pet.h --- + #pragma once + #include + + struct Pet { + std::string name; + std::string sound; + + Pet(std::string _name, std::string _sound) + : name(std::move(_name)), sound(std::move(_sound)) {} + + std::string speak() const { return name + " goes " + sound + "!"; } + }; + + // --- pets.cc --- + #include + #include + #include "pet.h" + + NB_MODULE(pets, m) { + auto pet = nanobind::class_(m, "Pet") + .def(nanobind::init()) + .def("speak", &Pet::speak); + + nanobind::export_for_interop(pet); + } + + // --- groomer.cc --- + #include + #include "pet.h" + + std::string groom(const Pet& pet) { + return pet.name + " got a haircut"; + } + + PYBIND11_MODULE(groomer, m) { + auto pet = pybind11::module_::import_("pets").attr("Pet"); + + // This could go either before or after the function definition that + // relies on it + pybind11::import_for_interop(pet); + + // If Pet were bound by a non-C++ framework, you would instead say: + // pybind11::import_for_interop(pet); + + m.def("groom", &groom); + } + + +Automatic communication +----------------------- + +In large binding projects, you might prefer to share *all* types rather than +only those you nominate. For that, pybind11 provides the +``py::interoperate_by_default()`` function. It takes two optional bool +parameters that specify whether you want automatic export and/or automatic +import; if you don't specify the parameters, then both are enabled. + +Automatic export is equivalent to writing a call to ``py::export_for_interop()`` +after every ``py::class_``, ``py::enum_``, or ``py::native_enum`` binding +statement in any pybind11 module that is ABI-compatible with the one in which +you wrote the call. + +Automatic import is equivalent to writing a call to ``py::import_for_interop()`` +after every export of a type from a different framework. It only import +bindings written in C++ with a compatible platform ABI (the same ones that +``py::import_for_interop()`` can import without a template argument); +bindings written in other languages must always be imported explicitly. + +Automatic import and export apply both to types that already exist and +types that will be bound in the future. They cannot be disabled once enabled. + +Here is the above example recast to use automatic communication. + +.. code-block:: cpp + + // (pet.h unchanged) + + // --- pets.cc --- + #include + #include + #include "pet.h" + + NB_MODULE(pets, m) { + nanobind::interoperate_by_default(); + nanobind::class_(m, "Pet") + .def(nanobind::init()) + .def("speak", &Pet::speak); + } + + // --- groomer.cc --- + #include + #include "pet.h" + + std::string groom(const Pet& pet) { + return pet.name + " got a haircut"; + } + + PYBIND11_MODULE(groomer, m) { + pybind11::interoperate_by_default(); + m.def("groom", &groom); + } + + +Conversion semantics and caveats +-------------------------------- + +Cross-framework inheritance is not supported: a type bound +using pybind11 must only have base classes that were bound using +ABI-compatible versions of pybind11. + +A function bound using pybind11 cannot perform a conversion to +``std::unique_ptr`` using a foreign binding for ``T``, because the +interoperability mechanism doesn't provide any way to ask a foreign instance +to relinquish its ownership. + +When converting from a foreign instance to ``std::shared_ptr``, pybind11 +generally cannot "see inside" the instance to find an existing ``shared_ptr`` +to share ownership with, so it will create a new ``shared_ptr`` control block +that owns a reference to the Python object. This is usually not a problem, but +does mean that ``shared_ptr::use_count()`` won't work like you expect. (If +``T`` inherits ``std::enable_shared_from_this``, then pybind11 can use that +to find the existing ``shared_ptr``, and will do so instead.) + +Type casters (both :ref:`built-in ` and :ref:`custom +`) execute before the interoperability mechanism +has a chance to step in. pybind11 is not able to execute type casters from +a different framework; you will need to port them to a pybind11 equivalent. +Interoperability only helps with bindings, as produced by ``py::class_`` and +similar statements. + +:ref:`Implicit conversion ` defined using +``py::implicitly_convertible()`` can convert *from* foreign types. +Implicit conversions *to* a foreign type should be registered with its +binding library, not with pybind11. + +When a C++-to-foreign-Python conversion is performed in a context that does +not specify the ``return_value_policy``, the policy to use is inferred using +pybind11's rules, which may differ from the foreign framework's. + +It is possible for multiple foreign bindings to exist for the same C++ type, +or for a particular C++ type to have both a native pybind11 binding +and one or more foreign ones. This might occur due to separate Python +extensions each having their own need to bind a common type, as discussed in +the section on :ref:`module-local bindings `. In such cases, +pybind11 always tries bindings for a given C++ type ``T`` in the following order: + +* the pybind11 binding for ``T`` that was declared with ``py::module_local()`` + in this extension module, if any; then + +* the pybind11 binding for ``T`` that was declared without ``py::module_local()`` + in either this extension module or another ABI-compatible one (drawing no + distinction between the two), if any; then + +* if performing a from-Python conversion on an instance of a pybind11 binding + for ``T`` that was declared with ``py::module_local()`` in a different + but ABI-compatible module, that binding; otherwise + +* each known foreign binding, in the order in which they were imported, + without making any distinction between other versions of pybind11 and + non-pybind11 frameworks. (If automatic import is enabled, then the import + order will match the original export order.) + +You can use the interoperability mechanism to share :ref:`module-local bindings +` with other modules. Unlike the sharing that happens by default, +this allows you to return instances of such bindings from outside the module in +which they were defined. + +When performing C++-to-Python conversion of a type for which +:ref:`automatic downcasting ` is applicable, +the downcast occurs in the binding library that is originally performing the +conversion, even if the result will then be obtained using a foreign binding. +That means foreign frameworks returning pybind11 types might not downcast +them in the same way that pybind11 does; they might only be able to downcast +from a primary base (with no this-pointer adjustment / no multiple inheritance), +or not downcast at all. diff --git a/docs/index.rst b/docs/index.rst index 77b097c574..42ef24aadb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,6 +34,7 @@ advanced/smart_ptrs advanced/cast/index advanced/pycpp/index + advanced/interop advanced/embedding advanced/misc advanced/deprecated diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 9b373fc268..71b1f83d47 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,55 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +v3.1 +==== + +The major new feature in pybind11 v3.1 is support for +:ref:`interoperability with other binding frameworks ` and other +(future) versions of pybind11. See the linked documentation for details. + +This support was added in an ABI-compatible way, so you can combine pybind11 +v3.1 extensions with v3.0 extensions. Classes and enums bound using pybind11 +v3.1 support all interoperability features. Classes and ``py::enum_``\s bound +using pybind11 v3.0 can still be exported manually by a pybind11 v3.1 extension +calling ``py::export_for_interop()``, but they won't be exported automatically +and they can't be returned by value from a foreign binding. +``py::native_enum``\s bound using pybind11 v3.0 don't support the +interoperability mechanism at all. + +There is one implication of the new interoperability support that might result +in new compiler errors for some previously-working binding code. Previously, +pybind11 only attempted to call a bound type's copy constructor or move +constructor if that type was ever returned from a pybind11-bound function. +Now, pybind11 must allow for the possibility that a pybind11-bound type is +returned from a foreign framework's bound functions, so it will generate +code that's capable of calling copy and move constructors for any bound type +that satisfies ``std::is_{copy,move}_constructible``. There exist types that +satisfy that type trait but will produce errors if you actually try to copy +them, such as the following: + +.. code-block:: cpp + + struct MoveOnly { MoveOnly(MoveOnly&&) noexcept = default; } + struct Container { + std::vector items; + }; + +``Container`` in this example satisfies ``std::is_copy_constructible``, but +actually trying to copy it will fail at compile time because the vector element +``MoveOnly`` is not copyable. The solution is to explicitly mark ``Container`` +as move-only: + +.. code-block:: cpp + + struct MoveOnly { MoveOnly(MoveOnly&&) noexcept = default; } + struct Container { + Container() = default; + Container(Container&&) noexcept = default; + + std::vector items; + }; + .. _upgrade-guide-3.0: v3.0 diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 09a81fa495..d4f7fd3dbe 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -84,21 +84,22 @@ class type_caster_enum_type { bool load(handle src, bool convert) { handle native_enum = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType))); - if (native_enum) { - if (!isinstance(src, native_enum)) { - return false; - } + if (native_enum && isinstance(src, native_enum)) { type_caster underlying_caster; if (!underlying_caster.load(src.attr("value"), convert)) { pybind11_fail("native_enum internal consistency failure."); } - value = static_cast(static_cast(underlying_caster)); + native_value = static_cast(static_cast(underlying_caster)); + native_loaded = true; return true; } - if (!pybind11_enum_) { - pybind11_enum_.reset(new type_caster_base()); + + type_caster_base legacy_caster; + if (legacy_caster.load(src, convert)) { + legacy_ptr = static_cast(legacy_caster); + return true; } - return pybind11_enum_->load(src, convert); + return false; } template @@ -106,23 +107,19 @@ class type_caster_enum_type { // NOLINTNEXTLINE(google-explicit-constructor) operator EnumType *() { - if (!pybind11_enum_) { - return &value; - } - return pybind11_enum_->operator EnumType *(); + return native_loaded ? &native_value : legacy_ptr; } // NOLINTNEXTLINE(google-explicit-constructor) operator EnumType &() { - if (!pybind11_enum_) { - return value; - } - return pybind11_enum_->operator EnumType &(); + return native_loaded ? native_value : + legacy_ptr ? *legacy_ptr : throw reference_cast_error(); } private: - std::unique_ptr> pybind11_enum_; - EnumType value; + EnumType native_value; // if loading a py::native_enum + bool native_loaded = false; + EnumType *legacy_ptr = nullptr; // if loading a py::enum_ or foreign }; template @@ -876,9 +873,9 @@ struct holder_caster_foreign_helpers { PyObject *o; }; - template - static bool try_shared_from_this(std::enable_shared_from_this *value, - std::shared_ptr *holder_out) { + template + static auto try_shared_from_this(type *value, std::shared_ptr *holder_out) + -> decltype(value->shared_from_this(), bool()) { // object derives from enable_shared_from_this; // try to reuse an existing shared_ptr if one is known if (auto existing = try_get_shared_from_this(value)) { @@ -911,9 +908,9 @@ struct holder_caster_foreign_helpers { template static bool - set_foreign_holder(handle src, type *value, std::shared_ptr *holder_out) { + set_foreign_holder(handle src, const type *value, std::shared_ptr *holder_out) { std::shared_ptr holder_mut; - if (set_foreign_holder(src, value, &holder_mut)) { + if (set_foreign_holder(src, const_cast(value), &holder_mut)) { *holder_out = holder_mut; return true; } diff --git a/include/pybind11/contrib/pymetabind.h b/include/pybind11/contrib/pymetabind.h index a4a7cca43f..1b7c52efe3 100644 --- a/include/pybind11/contrib/pymetabind.h +++ b/include/pybind11/contrib/pymetabind.h @@ -6,15 +6,20 @@ * This functionality is intended to be used by the framework itself, * rather than by users of the framework. * - * This is version 0.2+dev of pymetabind. Changelog: + * This is version 0.3 of pymetabind. Changelog: * - * Unreleased: Don't do a Py_DECREF in `pymb_remove_framework` since the - * interpreter might already be finalized at that point. + * Version 0.3: Don't do a Py_DECREF in `pymb_remove_framework` since the + * 2025-09-15 interpreter might already be finalized at that point. * Revamp binding lifetime logic. Add `remove_local_binding` * and `free_local_binding` callbacks. * Add `pymb_framework::registry` and use it to simplify * the signatures of `pymb_remove_framework`, * `pymb_add_binding`, and `pymb_remove_binding`. + * Update `to_python` protocol to be friendlier to + * pybind11 instances with shared/smart holders. + * Remove `pymb_rv_policy_reference_internal`; add + * `pymb_rv_policy_share_ownership`. Change `keep_alive` + * return value convention. * * Version 0.2: Use a bitmask for `pymb_framework::flags` and add leak_safe * 2025-09-11 flag. Change `translate_exception` to be non-throwing. @@ -98,33 +103,36 @@ extern "C" { #endif /* - * Approach used to cast a previously unknown C++ instance into a Python object. - * The values of these enumerators match those for `nanobind::rv_policy` and - * `pybind11::return_value_policy`. + * Approach used to cast a previously unknown native instance into a Python + * object. This is similar to `pybind11::return_value_policy` or + * `nanobind::rv_policy`; some different options are provided than those, + * but same-named enumerators have the same semantics and values. */ enum pymb_rv_policy { - // (Values 0 and 1 correspond to `automatic` and `automatic_reference`, - // which should become one of the other policies before reaching us) - - // Create a Python object that owns a pointer to heap-allocated storage - // and will destroy and deallocate it when the Python object is destroyed + // Create a Python object that wraps a pointer to a heap-allocated + // native instance and will destroy and deallocate it (in whatever way + // is most natural for the target language) when the Python object is + // destroyed pymb_rv_policy_take_ownership = 2, - // Create a Python object that owns a new C++ instance created via - // copy construction from the given one + // Create a Python object that owns a new native instance created by + // copying the given one pymb_rv_policy_copy = 3, - // Create a Python object that owns a new C++ instance created via - // move construction from the given one + // Create a Python object that owns a new native instance created by + // moving the given one pymb_rv_policy_move = 4, - // Create a Python object that wraps the given pointer to a C++ instance + // Create a Python object that wraps a pointer to a native instance // but will not destroy or deallocate it pymb_rv_policy_reference = 5, - // `reference`, plus arrange for the given `parent` python object to - // live at least as long as the new object that wraps the pointer - pymb_rv_policy_reference_internal = 6, + // Create a Python object that wraps a pointer to a native instance + // and will perform a custom action when the Python object is destroyed. + // The custom action is specified using the first call to keep_alive() + // after the object is created, and such a call must occur in order for + // the object to be considered fully initialized. + pymb_rv_policy_share_ownership = 6, // Don't create a new Python object; only try to look up an existing one // from the same framework @@ -272,6 +280,23 @@ enum pymb_framework_flags { pymb_framework_leak_safe = 0x0002, }; +/* Additional results from `pymb_framework::to_python` */ +struct pymb_to_python_feedback { + // Ignored on entry. On exit, set to 1 if the returned Python object + // was created by the `to_python` call, or zero if it already existed and + // was simply looked up. + uint8_t is_new; + + // On entry, indicates whether the caller can control whether the native + // instance `cobj` passed to `to_python` is destroyed after the conversion: + // set to 1 if a relocation is allowable or 0 if `cobj` must be destroyed + // after the call. (This is only relevant when using pymb_rv_policy_move.) + // On exit, set to 1 if destruction should be inhibited because `*cobj` + // was relocated into the new instance. Must be left as zero on exit if + // set to zero on entry. + uint8_t relocate; +}; + /* * Information about one framework that has registered itself with pymetabind. * "Framework" here refers to a set of bindings that are natively mutually @@ -346,8 +371,8 @@ struct pymb_framework { // The function pointers below allow other frameworks to interact with // bindings provided by this framework. They are constant after construction - // and must not throw C++ exceptions. Unless otherwise documented, - // they must not be NULL. + // and must not throw C++ exceptions. They must not be NULL; if a feature + // is not relevant to your use case, provide a stub that always fails. // Extract a C/C++/etc object from `pyobj`. The desired type is specified by // providing a `pymb_binding*` for some binding that belongs to this @@ -375,9 +400,12 @@ struct pymb_framework { // a copy of the object to which `from_python`'s return value points before // you drop the references. // - // On free-threaded builds, callers must ensure that the `binding` is not - // destroyed during a call to `from_python`. The requirements for this are - // subtle; see the full discussion in the comment for `struct pymb_binding`. + // On free-threaded builds, no direct synchronization is required to call + // this method, but you must ensure the `binding` won't be destroyed during + // (or before) your call. This generally requires maintaining a continuously + // attached Python thread state whenever you hold a pointer to `binding` + // that a concurrent call to your framework's `remove_foreign_binding` + // method wouldn't be able to clear. See the comment for `pymb_binding`. void* (*from_python)(struct pymb_binding* binding, PyObject* pyobj, uint8_t convert, @@ -386,30 +414,45 @@ struct pymb_framework { // Wrap the C/C++/etc object `cobj` into a Python object using the given // return value policy. The type is specified by providing a `pymb_binding*` - // for some binding that belongs to this framework. `parent` is relevant - // only if `rvp == pymb_rv_policy_reference_internal`. rvp must be one of - // the defined enumerators. Returns NULL if the cast is not possible, or - // a new reference otherwise. + // for some binding that belongs to this framework. + // + // The semantics of this function are as follows: + // - If there is already a live Python object created by this framework for + // this C++ object address and type, it will be returned and the `rvp` is + // ignored. + // - Otherwise, if `rvp == pymb_rv_policy_none`, NULL is returned without + // the Python error indicator set. + // - Otherwise, a new Python object will be created and returned. It will + // wrap either the pointer `cobj` or a copy/move of the contents of + // `cobj`, depending on the value of `rvp`. + // + // Returns a new reference to a Python object, or NULL if not possible. + // Also sets *feedback to provide additional information about the + // conversion. // - // A NULL return may leave the Python error indicator set if something - // specifically describable went wrong during conversion, but is not - // required to; returning NULL without PyErr_Occurred() should be - // interpreted as a generic failure to convert `cobj` to a Python object. + // After a successful `to_python` call that returns a new instance and + // used `pymb_rv_policy_share_ownership`, the caller must make a call to + // `keep_alive` to describe how the shared ownership should be managed. // - // On free-threaded builds, callers must ensure that the `binding` is not - // destroyed during a call to `to_python`. The requirements for this are - // subtle; see the full discussion in the comment for `struct pymb_binding`. + // On free-threaded builds, no direct synchronization is required to call + // this method, but you must ensure the `binding` won't be destroyed during + // (or before) your call. This generally requires maintaining a continuously + // attached Python thread state whenever you hold a pointer to `binding` + // that a concurrent call to your framework's `remove_foreign_binding` + // method wouldn't be able to clear. See the comment for `pymb_binding`. PyObject* (*to_python)(struct pymb_binding* binding, void* cobj, enum pymb_rv_policy rvp, - PyObject* parent) PYMB_NOEXCEPT; + struct pymb_to_python_feedback* feedback) PYMB_NOEXCEPT; // Request that a PyObject reference be dropped, or that a callback // be invoked, when `nurse` is destroyed. `nurse` should be an object // whose type is bound by this framework. If `cb` is NULL, then // `payload` is a PyObject* to decref; otherwise `payload` will - // be passed as the argument to `cb`. Returns 0 if successful, - // or -1 and sets the Python error indicator on error. + // be passed as the argument to `cb`. Returns 1 if successful, + // 0 on error. This method may always return 0 if the framework has + // no better way to do a keep-alive than by creating a weakref; + // it is expected that the caller can handle creating the weakref. // // No synchronization is required to call this method. int (*keep_alive)(PyObject* nurse, @@ -424,8 +467,8 @@ struct pymb_framework { // such as `std::exception`. If translation succeeds, return 1 with the // Python error indicator set; otherwise, return 0. An exception may be // converted into a different exception by modifying `*eptr` and returning - // zero. This function pointer may be NULL if this framework does not - // provide exception translation. + // zero. This method may be set to NULL if its framework does not have + // a concept of exception translation. // // No synchronization is required to call this method. int (*translate_exception)(void* eptr) PYMB_NOEXCEPT; diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 485994fdac..543c6dcb6d 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -274,6 +274,23 @@ # endif #endif +// 3.14 Compatibility +#if !defined(Py_GIL_DISABLED) +inline bool is_uniquely_referenced(PyObject *obj) { + return Py_REFCNT(obj) == 1; +} +#elif 0x030E0000 <= PY_VERSION_HEX +inline bool is_uniquely_referenced(PyObject *obj) { + return PyUnstable_Object_IsUniquelyReferenced(obj); +} +#else // backport for 3.13 +inline bool is_uniquely_referenced(PyObject *obj) { + return _Py_IsOwnedByCurrentThread(obj) && + _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 && + _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0; +} +#endif + // 3.13 Compatibility #if 0x030D0000 <= PY_VERSION_HEX # define PYBIND11_TYPE_IS_TYPE_HINT "typing.TypeIs" @@ -1351,5 +1368,15 @@ constexpr # define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET #endif +#if defined(PY_BIG_ENDIAN) +# define PYBIND11_BIG_ENDIAN PY_BIG_ENDIAN +#else // pypy doesn't define PY_BIG_ENDIAN +# if defined(_MSC_VER) +# define PYBIND11_BIG_ENDIAN 0 // All Windows platforms are little-endian +# else // GCC and Clang define the following macros +# define PYBIND11_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# endif +#endif + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index df378b59f6..f28f74a5b4 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -22,11 +22,8 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { auto &interop_internals = get_interop_internals(); for (pymb_framework *fw : interop_internals.exc_frameworks) { - try { - fw->translate_exception(&p); - } catch (...) { - p = std::current_exception(); - } + if (fw->translate_exception(&p)) + return; } std::rethrow_exception(p); } @@ -38,12 +35,25 @@ inline bool should_autoimport_foreign(interop_internals &interop_internals, && binding->framework->abi_extra == interop_internals.self->abi_extra; } +// Determine whether a pybind11 type is module-local from a different module +inline bool is_local_to_other_module(type_info *ti) { + return ti->module_local_load != nullptr && + ti->module_local_load != &type_caster_generic::local_load; +} + // Add the given `binding` to our type maps so that we can use it to satisfy // from- and to-Python requests for the given C++ type inline void import_foreign_binding(pymb_binding *binding, const std::type_info *cpptype) noexcept { // Caller must hold the internals lock auto &interop_internals = get_interop_internals(); interop_internals.imported_any = true; + auto range = interop_internals.bindings.equal_range(*cpptype); + for (auto it = range.first; it != range.second; ++it) { + if (it->second == binding) { + return; // already imported + } + } + ++interop_internals.bindings_update_count; interop_internals.bindings.emplace(*cpptype, binding); } @@ -55,6 +65,35 @@ inline void *interop_cb_from_python(pymb_binding *binding, uint8_t convert, void (*keep_referenced)(void *ctx, PyObject *obj), void *keep_referenced_ctx) noexcept { + if (binding->context == nullptr) { + // This is a native enum type. We can only return a pointer to the C++ + // enum if we're able to allocate a temporary. + handle pytype((PyObject *) binding->pytype); + if (!keep_referenced || !isinstance(pyobj, pytype)) { + return nullptr; + } + try { + auto cap = reinterpret_borrow( + pytype.attr(native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + auto value = handle(pyobj).attr("value"); + uint64_t ival; + if (info->is_signed && handle(value) < int_(0)) { + ival = (uint64_t) cast(value); + } else { + ival = cast(value); + } + bytes holder{reinterpret_cast(&ival) + + PYBIND11_BIG_ENDIAN * (8 - info->size_bytes), + info->size_bytes}; + keep_referenced(keep_referenced_ctx, holder.ptr()); + return PyBytes_AsString(holder.ptr()); + } catch (error_already_set &exc) { + exc.discard_as_unraisable("Error converting native enum from Python"); + return nullptr; + } + } + #if defined(PYBIND11_HAS_OPTIONAL) using maybe_life_support = std::optional; #else @@ -85,7 +124,8 @@ inline void *interop_cb_from_python(pymb_binding *binding, type_caster_generic caster{static_cast(binding->context)}; void *ret = nullptr; try { - if (caster.load(pyobj, convert != 0)) { + if (caster.load_impl(pyobj, convert != 0, + /* foreign_ok */ false)) { ret = caster.value; } } catch (...) { @@ -101,18 +141,109 @@ inline void *interop_cb_from_python(pymb_binding *binding, return ret; } +// This wraps the call to type_info::init_instance() in some cases when casting +// a pybind11-bound object to Python on behalf of a foreign framework. It +// inhibits registration of the new instance so that interop_cb_keep_alive() +// can fix up the holder before other threads start using the new instance. +inline void init_instance_unregistered(instance *inst, const void *holder) { + assert(holder == nullptr && !inst->owned); + value_and_holder v_h = *values_and_holders(inst).begin(); + + // If using smart_holder, force creation of a shared_ptr that has a + // guarded_delete deleter, so that we can modify it in + // interop_cb_keep_alive(). We can't create it there because it needs to be + // created in the same DSO as it's accessed in; init_instance is in that + // DSO, but this function might not be. + if (v_h.type->holder_enum_v == holder_enum_t::smart_holder) { + inst->owned = true; + } + + // Pretend it's already registered so that init_instance doesn't try again + v_h.set_instance_registered(true); + + // Undo our shenanigans even if init_instance raises an exception + struct guard { + value_and_holder& v_h; + ~guard() noexcept { + v_h.set_instance_registered(false); + if (v_h.type->holder_enum_v == holder_enum_t::smart_holder) { + v_h.inst->owned = false; + auto &h = v_h.holder(); + h.vptr_is_using_std_default_delete = true; + h.reset_vptr_deleter_armed_flag( + v_h.type->get_memory_guarded_delete, /* armed_flag */ false); + } + } + } guard{v_h}; + v_h.type->init_instance(inst, nullptr); +} + inline PyObject *interop_cb_to_python(pymb_binding *binding, void *cobj, enum pymb_rv_policy rvp_, - PyObject *parent) noexcept { - const auto *ti = static_cast(binding->context); + pymb_to_python_feedback *feedback) noexcept { + feedback->relocate = 0; // we don't support relocation + feedback->is_new = 0; // unless overridden below + if (cobj == nullptr) { return none().release().ptr(); } - auto rvp = static_cast(rvp_); - if (rvp > return_value_policy::reference_internal) { - // Treat out-of-range rvp as "return existing instance but don't - // make a new one", for compatibility with pymb_rv_policy_none + + if (!binding->context) { + // Native enum type + try { + handle pytype((PyObject *) binding->pytype); + auto cap = reinterpret_borrow( + pytype.attr(native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + uint64_t key; + switch (info->size_bytes) { + case 1: key = *(uint8_t *) cobj; break; + case 2: key = *(uint16_t *) cobj; break; + case 4: key = *(uint32_t *) cobj; break; + case 8: key = *(uint64_t *) cobj; break; + default: return nullptr; + } + if (rvp_ == pymb_rv_policy_take_ownership) + ::operator delete(cobj); + if (info->is_signed) { + int64_t ikey = (int64_t) key; + if (info->size_bytes < 8) { + // sign extend + ikey <<= (64 - (info->size_bytes * 8)); + ikey >>= (64 - (info->size_bytes * 8)); + } + return pytype(ikey).release().ptr(); + } + return pytype(key).release().ptr(); + } catch (error_already_set& exc) { + exc.restore(); + return nullptr; + } + } + + const auto *ti = static_cast(binding->context); + return_value_policy rvp = return_value_policy::automatic; + bool inhibit_registration = false; + + switch (rvp_) { + case pymb_rv_policy_take_ownership: + case pymb_rv_policy_copy: + case pymb_rv_policy_move: + case pymb_rv_policy_reference: + // These have the same values and semantics as our own policies + rvp = (return_value_policy) rvp_; + break; + case pymb_rv_policy_share_ownership: + rvp = return_value_policy::reference; + inhibit_registration = true; + break; + case pymb_rv_policy_none: + break; + } + if (rvp == return_value_policy::automatic) { + // Specified rvp was none, or was something unrecognized so we should + // be conservative and treat it like none. return find_registered_python_instance(cobj, ti).ptr(); } @@ -128,7 +259,13 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, } try { - return type_caster_generic::cast(cobj, rvp, parent, ti, copy_ctor, move_ctor).ptr(); + type_caster_generic::cast_sources srcs{cobj, ti}; + if (inhibit_registration) { + srcs.init_instance = init_instance_unregistered; + } + handle ret = type_caster_generic::cast(srcs, rvp, {}, copy_ctor, move_ctor); + feedback->is_new = srcs.is_new; + return ret.ptr(); } catch (...) { translate_exception(std::current_exception()); return nullptr; @@ -137,37 +274,75 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, inline int interop_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void *)) noexcept { try { + do { // single-iteration loop to reduce nesting level + if (!is_uniquely_referenced(nurse)) { + break; + } + // See if we can install this as a shared_ptr deleter rather than + // a keep_alive, since the very first keep_alive for a new object + // might be to let it carry shared_ptr ownership. This helps + // a shared_ptr returned from a foreign binding be acceptable + // as a shared_ptr argument to a pybind11-bound function. + values_and_holders vhs{nurse}; + if (vhs.size() != 1) { + break; + } + value_and_holder v_h = *vhs.begin(); + if (v_h.instance_registered()) { + break; + } + auto cb_to_use = cb ? cb : (decltype(cb)) Py_DecRef; + bool success = false; + if (v_h.type->holder_enum_v == holder_enum_t::std_shared_ptr && + !v_h.holder_constructed()) { + // Create a shared_ptr whose destruction will perform the action + std::shared_ptr owner(payload, cb_to_use); + // Use the aliasing constructor to make its get() return the right thing + new (std::addressof(v_h.holder>())) std::shared_ptr( + std::move(owner), v_h.value_ptr()); + v_h.set_holder_constructed(); + success = true; + } else if (v_h.type->holder_enum_v == holder_enum_t::smart_holder && + v_h.holder_constructed() && !v_h.inst->owned) { + auto &h = v_h.holder(); + auto *gd = v_h.type->get_memory_guarded_delete(h.vptr); + if (gd && !gd->armed_flag) { + gd->del_fun = [=](void*) { cb_to_use(payload); }; + gd->use_del_fun = true; + gd->armed_flag = true; + success = true; + } + } + register_instance(v_h.inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(true); + if (success) { + return 1; + } + } while (false); + if (!cb) { keep_alive_impl(nurse, static_cast(payload)); } else { capsule patient{payload, cb}; keep_alive_impl(nurse, patient); } - return 0; + return 1; } catch (...) { translate_exception(std::current_exception()); - return -1; + PyErr_WriteUnraisable(nurse); + return 0; } } -inline void interop_cb_translate_exception(const void *eptr) { - with_exception_translators( +inline int interop_cb_translate_exception(void *eptr) noexcept { + return with_exception_translators( [&](std::forward_list &exception_translators, - std::forward_list &local_exception_translators) { - // Try local translators. These don't have any special entries - // we need to skip. - std::exception_ptr e = *(const std::exception_ptr *) eptr; - for (auto &translator : local_exception_translators) { - try { - translator(e); - return; - } catch (...) { - e = std::current_exception(); - } - } - + std::forward_list & /*local_exception_translators*/) { + // Ignore local exception translators. We're being called to translate + // an exception that was raised from a different framework, thus a + // different extension module, so nothing local to us will apply. // Try global translators, except the last one or two. - e = *(const std::exception_ptr *) eptr; + std::exception_ptr &e = *(std::exception_ptr *) eptr; auto it = exception_translators.begin(); auto leader = it; // - The last one is the default translator. It translates @@ -186,7 +361,7 @@ inline void interop_cb_translate_exception(const void *eptr) { for (; leader != exception_translators.end(); ++it, ++leader) { try { (*it)(e); - return; + return 1; } catch (...) { e = std::current_exception(); } @@ -198,7 +373,7 @@ inline void interop_cb_translate_exception(const void *eptr) { } catch (error_already_set &err) { handle_nested_exception(err, e); err.restore(); - return; + return 1; } catch (const builtin_exception &err) { // Could not use template since it's an abstract class. if (const auto *nep @@ -206,13 +381,35 @@ inline void interop_cb_translate_exception(const void *eptr) { handle_nested_exception(*nep, e); } err.set_error(); - return; + return 1; + } catch (...) { + e = std::current_exception(); } - // Anything not caught by the above bubbles out. + return 0; }); } inline void interop_cb_remove_local_binding(pymb_binding *binding) noexcept { + with_internals([&](internals &) { + auto &interop_internals = get_interop_internals(); + auto *cpptype = (const std::type_info *) binding->native_type; + auto range = interop_internals.bindings.equal_range(*cpptype); + for (auto it = range.first; it != range.second; ++it) { + if (it->second == binding) { + ++interop_internals.bindings_update_count; + interop_internals.bindings.erase(it); + return; + } + } + }); +} + +inline void interop_cb_free_local_binding(pymb_binding *binding) noexcept { + free(const_cast(binding->source_name)); + delete binding; +} + +inline void interop_cb_add_foreign_binding(pymb_binding *binding) noexcept { with_internals([&](internals &) { auto &interop_internals = get_interop_internals(); if (should_autoimport_foreign(interop_internals, binding)) { @@ -228,6 +425,7 @@ inline void interop_cb_remove_foreign_binding(pymb_binding *binding) noexcept { auto range = interop_internals.bindings.equal_range(*type); for (auto it = range.first; it != range.second; ++it) { if (it->second == binding) { + ++interop_internals.bindings_update_count; interop_internals.bindings.erase(it); break; } @@ -265,22 +463,35 @@ inline void interop_cb_add_foreign_framework(pymb_framework *framework) noexcept exception_translators.insert_after(trailer, foreign_exception_translator); } // Add the new framework at the end of the list - auto leader = interop_internals.exc_frameworks.begin(); - auto trailer = leader; - while (++leader != interop_internals.exc_frameworks.end()) { - ++trailer; + auto it = interop_internals.exc_frameworks.before_begin(); + while (std::next(it) != interop_internals.exc_frameworks.end()) { + ++it; } - interop_internals.exc_frameworks.insert_after(trailer, framework); + interop_internals.exc_frameworks.insert_after(it, framework); }); } } +inline void interop_cb_remove_foreign_framework(pymb_framework *framework) noexcept { + // No need for locking; the interpreter is already finalizing + // at this point (and might be already finalized, so we can't do any + // Python API calls) + if (framework->translate_exception) { + get_interop_internals().exc_frameworks.remove(framework); + // No need to bother removing the foreign_exception_translator if + // this was the last of the exc_frameworks. In the unlikely event + // that something needs an exception translated during finalization, + // it will work fine with an empty exc_frameworks list. + } +} + // (end of callbacks) // Advertise our existence, and the above callbacks, to other frameworks PYBIND11_NOINLINE bool interop_internals::initialize() { + pymb_registry *registry = nullptr; bool inited_by_us = with_internals([&](internals &) { - if (registry) { + if (self) { return false; } registry = pymb_get_registry(); @@ -290,17 +501,19 @@ PYBIND11_NOINLINE bool interop_internals::initialize() { self.reset(new pymb_framework{}); self->name = "pybind11 " PYBIND11_ABI_TAG; - self->bindings_usable_forever = 0; - self->leak_safe = 0; + self->flags = 0; self->abi_lang = pymb_abi_lang_cpp; self->abi_extra = PYBIND11_PLATFORM_ABI_ID; self->from_python = interop_cb_from_python; self->to_python = interop_cb_to_python; self->keep_alive = interop_cb_keep_alive; self->translate_exception = interop_cb_translate_exception; + self->remove_local_binding = interop_cb_remove_local_binding; + self->free_local_binding = interop_cb_free_local_binding; self->add_foreign_binding = interop_cb_add_foreign_binding; self->remove_foreign_binding = interop_cb_remove_foreign_binding; self->add_foreign_framework = interop_cb_add_foreign_framework; + self->remove_foreign_framework = interop_cb_remove_foreign_framework; return true; }); if (inited_by_us) { @@ -311,7 +524,11 @@ PYBIND11_NOINLINE bool interop_internals::initialize() { return inited_by_us; } -inline interop_internals::~interop_internals() = default; +inline interop_internals::~interop_internals() { + if (self && bindings.empty()) { + pymb_remove_framework(self.get()); + } +} // Learn to satisfy from- and to-Python requests for `cpptype` using the // foreign binding provided by the given `pytype`. If cpptype is nullptr, infer @@ -325,8 +542,28 @@ PYBIND11_NOINLINE void import_for_interop(handle pytype, const std::type_info *c pybind11_fail("pybind11::import_for_interop(): type does not define " "a __pymetabind_binding__"); } + if (binding->pytype != (PyTypeObject *) pytype.ptr()) { + pybind11_fail("pybind11::import_for_interop(): the binding associated " + "with the type you specified is for a different type; " + "pass the type object that was created by the other " + "framework, not its subclass"); + } if (binding->framework == interop_internals.self.get()) { - pybind11_fail("pybind11::import_for_interop(): type is not foreign"); + // Can't call get_type_info() because it would lock internals and + // they're already locked + auto &internals = get_internals(); + auto it = internals.registered_types_py.find(binding->pytype); + if (it != internals.registered_types_py.end() && + it->second.size() == 1 && + is_local_to_other_module(*it->second.begin())) { + // Allow importing module-local types from other pybind11 modules, + // even if they're ABI-compatible with us and thus use the same + // pymb_framework. The import is not doing much here; the export + // alone would put the binding in interop_internals where we can + // see it. + } else { + pybind11_fail("pybind11::import_for_interop(): type is not foreign"); + } } if (!cpptype) { if (binding->framework->abi_lang != pymb_abi_lang_cpp) { @@ -376,47 +613,43 @@ PYBIND11_NOINLINE void interop_enable_import_all() { // we can reuse the pymb callback functions. interop_internals registry + // self never change once they're non-null, so we can access them // without locking here. - pymb_lock_registry(interop_internals.registry); + struct pymb_registry *registry = interop_internals.self->registry; + pymb_lock_registry(registry); // NOLINTNEXTLINE(modernize-use-auto) - PYMB_LIST_FOREACH(struct pymb_binding *, binding, interop_internals.registry->bindings) { - if (binding->framework != interop_internals.self.get() - && pymb_try_ref_binding(binding) != 0) { + PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { + if (binding->framework != interop_internals.self.get()) { interop_cb_add_foreign_binding(binding); - pymb_unref_binding(binding); } } - pymb_unlock_registry(interop_internals.registry); + pymb_unlock_registry(registry); } // Expose hooks for other frameworks to use to work with the given pybind11 -// type object. Caller must hold the internals lock and have already called +// type object. `ti` may be nullptr if exporting a native enum. +// Caller must hold the internals lock and have already called // interop_internals.initialize_if_needed(). -PYBIND11_NOINLINE void export_for_interop(type_info *ti) { +PYBIND11_NOINLINE void export_for_interop(const std::type_info *cpptype, + PyTypeObject *pytype, + type_info *ti) { auto &interop_internals = get_interop_internals(); - auto range = interop_internals.bindings.equal_range(*ti->cpptype); + auto range = interop_internals.bindings.equal_range(*cpptype); for (auto it = range.first; it != range.second; ++it) { - if (it->second->framework == interop_internals.self.get()) { + if (it->second->framework == interop_internals.self.get() && + it->second->pytype == pytype) { return; // already exported } } auto *binding = new pymb_binding{}; binding->framework = interop_internals.self.get(); - binding->pytype = ti->type; - binding->native_type = ti->cpptype; - binding->source_name = PYBIND11_COMPAT_STRDUP(clean_type_id(ti->cpptype->name()).c_str()); + binding->pytype = pytype; + binding->native_type = cpptype; + binding->source_name = PYBIND11_COMPAT_STRDUP(clean_type_id(cpptype->name()).c_str()); binding->context = ti; - capsule tie_lifetimes((void *) binding, [](void *p) { - auto *binding = (pymb_binding *) p; - pymb_remove_binding(get_interop_internals().registry, binding); - free(const_cast(binding->source_name)); - delete binding; - }); - keep_alive_impl((PyObject *) ti->type, tie_lifetimes); - - interop_internals.bindings.emplace(*ti->cpptype, binding); - pymb_add_binding(interop_internals.registry, binding); + ++interop_internals.bindings_update_count; + interop_internals.bindings.emplace(*cpptype, binding); + pymb_add_binding(binding, /* tp_finalize_will_remove */ 0); } // Call `export_type_to_foreign()` for each type that currently exists in our @@ -437,7 +670,21 @@ PYBIND11_NOINLINE void interop_enable_export_all() { interop_internals.initialize_if_needed(); with_internals([&](internals &internals) { for (const auto &entry : internals.registered_types_cpp) { - detail::export_for_interop(entry.second); + auto *ti = entry.second; + detail::export_for_interop(ti->cpptype, ti->type, ti); + } + for (const auto &entry : internals.native_enum_type_map) { + try { + auto cap = reinterpret_borrow( + handle(entry.second).attr(native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + detail::export_for_interop(info->cpptype, + (PyTypeObject *) entry.second, + nullptr); + } catch (error_already_set&) { + // Ignore native enums without a __pybind11_enum__ capsule; + // they might be from an older version of pybind11 + } } }); } @@ -451,85 +698,53 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, void *closure) { auto &internals = get_internals(); auto &interop_internals = get_interop_internals(); + uint32_t update_count = interop_internals.bindings_update_count; + + do { + PYBIND11_LOCK_INTERNALS(internals); + (void) internals; // suppress unused warning on non-ft builds + auto range = interop_internals.bindings.equal_range(*type); + auto it = range.first; + for (; it != range.second; ++it) { + auto *binding = it->second; + if (binding->framework == interop_internals.self.get() && + (!binding->context || + !is_local_to_other_module((type_info *) binding->context))) { + // Don't try to use our own types, unless they're module-local + // to some other module and this is the only way we'd see them. + // (The module-local escape hatch is only relevant for + // to-Python conversions; from-Python won't try foreign if it + // sees the capsule for other-module-local.) + continue; + } - PYBIND11_LOCK_INTERNALS(internals); - (void) internals; // suppress unused warning on non-ft builds - auto range = interop_internals.bindings.equal_range(*type); - - if (range.first == range.second) { - return nullptr; // no foreign bindings - } - - if (std::next(range.first) == range.second) { - // Single binding - check that it's not our own - auto *binding = range.first->second; - if (binding->framework != foreign_internals.self.get() - && pymb_try_ref_binding(binding) != 0) { #ifdef Py_GIL_DISABLED // attempt() might execute Python code; drop the internals lock // to avoid a deadlock lock.unlock(); #endif void *result = attempt(closure, binding); - pymb_unref_binding(binding); - return result; - } - return nullptr; - } - - // Multiple bindings - try all except our own -#ifndef Py_GIL_DISABLED - for (auto it = range.first; it != range.second; ++it) { - auto *binding = it->second; - if (binding->framework != interop_internals.self.get() - && pymb_try_ref_binding(binding) != 0) { - void *result = attempt(closure, binding); - pymb_unref_binding(binding); if (result) { return result; } +#ifdef Py_GIL_DISABLED + lock.lock(); +#endif + // Make sure our iterator wasn't invalidated by something that + // was done within attempt(), or concurrently during attempt() + // while we didn't hold the internals lock + if (interop_internals.bindings_update_count != update_count) { + // Concurrent update occurred; retry + update_count = interop_internals.bindings_update_count; + break; + } } - } - return nullptr; -#else - // In free-threaded mode, this is tricky: we need to drop the - // internals lock before calling attempt(), but once we do so, - // any of these bindings that might be in the middle of getting deleted - // can be concurrently removed from the map, which would interfere - // with our iteration. Copy the binding pointers out of the list to avoid - // this problem. - - // Count the number of foreign bindings we might see - size_t len = (size_t) std::distance(range.first, range.second); - - // Allocate temporary storage for that many pointers - pymb_binding **scratch = (pymb_binding **) alloca(len * sizeof(pymb_binding *)); - pymb_binding **scratch_tail = scratch; - - // Iterate again, taking out strong references and saving pointers to - // our scratch storage - for (auto it = range.first; it != range.second; ++it) { - auto *binding = it->second; - if (binding->framework != interop_internals.self.get() - && pymb_try_ref_binding(binding) != 0) { - *scratch_tail++ = binding; - } - } - - // Drop the lock and proceed using only our saved binding pointers. - // Since we obtained strong references to them, there is no remaining - // concurrent-destruction hazard. - lock.unlock(); - void *result = nullptr; - while (scratch != scratch_tail) { - if (!result) { - result = attempt(closure, *scratch); + if (it != range.second) { + // We broke out early due to a concurrent update. Retry from the top. + continue; } - pymb_unref_binding(*scratch); - ++scratch; - } - return result; -#endif + return nullptr; + } while (true); } PYBIND11_NAMESPACE_END(detail) @@ -545,7 +760,10 @@ inline void interoperate_by_default(bool export_all = true, bool import_all = tr } template -inline void import_for_interop(type pytype) { +inline void import_for_interop(handle pytype) { + if (!PyType_Check(pytype.ptr())) { + pybind11_fail("pybind11::import_for_interop(): expected a type object"); + } const std::type_info *cpptype = std::is_void::value ? nullptr : &typeid(T); auto &interop_internals = detail::get_interop_internals(); interop_internals.initialize_if_needed(); @@ -553,15 +771,41 @@ inline void import_for_interop(type pytype) { [&](detail::internals &) { detail::import_for_interop(std::move(pytype), cpptype); }); } -inline void export_for_interop(type ty) { +inline void export_for_interop(handle ty) { + if (!PyType_Check(ty.ptr())) { + pybind11_fail("pybind11::export_for_interop(): expected a type object"); + } auto &interop_internals = detail::get_interop_internals(); interop_internals.initialize_if_needed(); detail::type_info *ti = detail::get_type_info((PyTypeObject *) ty.ptr()); - if (!ti) { - pybind11_fail("pybind11::export_type_to_foreign: not a " - "pybind11 registered type"); + if (ti) { + detail::with_internals([&](detail::internals &) { + detail::export_for_interop(ti->cpptype, ti->type, ti); + }); + return; } - detail::with_internals([&](detail::internals &) { detail::export_for_interop(ti); }); + // Not a class_; maybe it's a native_enum? + try { + auto cap = reinterpret_borrow( + ty.attr(detail::native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + bool ours = detail::with_internals([&](detail::internals &internals) { + auto it = internals.native_enum_type_map.find(*info->cpptype); + if (it != internals.native_enum_type_map.end() && + it->second == ty.ptr()) { + detail::export_for_interop(info->cpptype, + (PyTypeObject *) ty.ptr(), + nullptr); + return true; + } + return false; + }); + if (ours) { + return; + } + } catch (error_already_set&) {} + pybind11_fail("pybind11::export_for_interop: not a " + "pybind11 class or enum bound in this domain"); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 79b2172142..bdeebdee2e 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -43,7 +43,8 @@ struct pymb_registry; /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -// REMINDER for next version bump: remove loader_life_support_tls +// REMINDER for next version bump: remove loader_life_support_tls + +// merge interop_internals into internals # define PYBIND11_INTERNALS_VERSION 11 #endif @@ -312,7 +313,8 @@ using copy_or_move_ctor = void *(*) (const void *); // indicating whether any foreign bindings are also known for its C++ type; // that way we can avoid an extra lookup when conversion to a native type fails. struct interop_internals { - // Registered foreign bindings for each C++ type. + // Registered pymetabind bindings for each C++ type, including both our + // own (exported) and other frameworks' (imported). // Protected by internals::mutex. type_multimap bindings; @@ -328,10 +330,6 @@ struct interop_internals { // translate_exception methods of each framework in this list. std::forward_list exc_frameworks; - // Pointer to the registry of foreign bindings. - // Protected by internals::mutex; constant once becoming non-null. - pymb_registry *registry = nullptr; - // Hooks allowing other frameworks to interact with us. // Protected by internals::mutex; constant once becoming non-null. std::unique_ptr self; @@ -347,7 +345,7 @@ struct interop_internals { // foreign.h. Instead, only compilation units that call // interoperate_by_default(), import_for_interop(), or // export_for_interop() will emit that code. - void (*export_for_interop)(type_info *); + void (*export_for_interop)(const std::type_info *, PyTypeObject *, type_info *); // Should we automatically advertise our types to other binding frameworks, // or only when requested via pybind11::export_for_interop()? @@ -364,11 +362,16 @@ struct interop_internals { // own types? bool imported_any = false; + // Number of times the `bindings` map has been modified. Used to detect + // cases where the iterator in try_foreign_bindings() may have been + // invalidated. Protected by internals::mutex. + uint32_t bindings_update_count = 0; + inline ~interop_internals(); // Returns true if we initialized, false if someone else already did. inline bool initialize_if_needed() { - if (registry) { + if (self) { return false; } return initialize(); diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index fe4ec82778..10c1a3e12d 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -211,6 +211,13 @@ inline void native_enum_data::finalize() { py_enum.attr(native_enum_info::attribute_name()) = enum_info; with_internals([&](internals &internals) { internals.native_enum_type_map[enum_type_index] = py_enum.ptr(); + + auto &interop_internals = get_interop_internals(); + if (interop_internals.export_all) { + auto info = enum_info.get_pointer(); + interop_internals.export_for_interop( + info->cpptype, (PyTypeObject *) py_enum.ptr(), nullptr); + } }); } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 8cfbfee33d..21256fe024 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -107,7 +107,7 @@ class loader_life_support { } static bool can_add_patient() { - return get_internals().loader_life_support_tls.get() != nullptr; + return tls_current_frame() != nullptr; } const std::unordered_set &list_patients() const { return keep_alive; } @@ -811,6 +811,7 @@ struct load_helper : value_and_holder_helper { } throw std::runtime_error("Non-owning holder (load_as_shared_ptr)."); } + auto *type_raw_ptr = static_cast(void_raw_ptr); if (python_instance_is_alias && !force_potentially_slicing_shared_ptr) { auto *vptr_gd_ptr = tinfo->get_memory_guarded_delete(holder().vptr); @@ -940,7 +941,11 @@ class type_caster_generic { // this does not provide enough information to use a foreign type or // to render a useful error message cast_sources(const void *obj, const detail::type_info *tinfo) - : original{obj, tinfo ? tinfo->cpptype : nullptr}, result{obj, tinfo} {} + : original{obj, tinfo ? tinfo->cpptype : nullptr}, result{obj, tinfo} { + if (tinfo) { + init_instance = tinfo->init_instance; + } + } // The object passed to cast(), with its static type. // original.type must not be null if resolve() will be called. @@ -954,14 +959,23 @@ class type_caster_generic { // The source to use for this cast, and the corresponding pybind11 // type_info. If the type_info is null, then pybind11 doesn't know - // about this type, but a foreign cast might work. + // about this type, but a foreign or other-module-local cast might work. std::pair result; // cast() sets this to the foreign framework used, if any mutable pymb_framework *used_foreign = nullptr; - // Returns true if the cast will not succeed using a type known to - // pybind11 (it will either use a foreign-framework type, or fail). + // Copy of type_info::init_instance, so it can be overridden in some + // cases casting a pybind11 instance on behalf of a foreign framework. + void (*init_instance)(instance *, const void *) = nullptr; + + // cast() sets this to indicate whether the cast created a new instance + // or looked up one that already existed. + mutable bool is_new = false; + + // Returns true if the cast will not succeed using a type listed in + // our pybind11 internals (it will either use a foreign-framework type + // or fail). bool needs_foreign() const { return result.second == nullptr; } // Returns true if the cast will use a pybind11 type that uses @@ -972,15 +986,17 @@ class type_caster_generic { } private: - PYBIND11_NOINLINE std::pair resolve() { + std::pair resolve() { if (downcast.type) { if (same_type(*original.type, *downcast.type)) { - const_cast(downcast.type) = nullptr; + downcast.type = nullptr; } else if (const auto *tpi = get_type_info(*downcast.type)) { + init_instance = tpi->init_instance; return {downcast.obj, tpi}; } } if (const auto *tpi = get_type_info(*original.type)) { + init_instance = tpi->init_instance; return {original.obj, tpi}; } return {nullptr, nullptr}; @@ -999,48 +1015,67 @@ class type_caster_generic { } static handle - cast_foreign(const cast_sources &srcs, return_value_policy policy, handle parent) { + cast_foreign(const cast_sources &srcs, + return_value_policy policy, + handle parent, + bool has_holder) { struct capture { const void *src; - pymb_rv_policy policy; - PyObject *parent; - pymb_framework **used_foreign; + pymb_rv_policy policy = pymb_rv_policy_none; + pymb_to_python_feedback feedback{}; + const cast_sources *srcs; } cap; auto attempt = +[](void *closure, pymb_binding *binding) -> void * { capture &cap = *(capture *) closure; void *ret = binding->framework->to_python( - binding, const_cast(cap.src), cap.policy, cap.parent); + binding, const_cast(cap.src), cap.policy, &cap.feedback); if (ret) { - *cap.used_foreign = binding->framework; + cap.srcs->used_foreign = binding->framework; + cap.srcs->is_new = cap.feedback.is_new; } return ret; }; - cap.parent = parent.ptr(); - cap.used_foreign = &srcs.used_foreign; + cap.srcs = &srcs; switch (policy) { case return_value_policy::automatic: cap.policy = pymb_rv_policy_take_ownership; break; case return_value_policy::automatic_reference: - cap.policy = pymb_rv_policy_reference; + case return_value_policy::reference: + cap.policy = has_holder ? pymb_rv_policy_share_ownership + : pymb_rv_policy_reference; break; - default: + case return_value_policy::reference_internal: + if (!parent) { + return nullptr; + } + cap.policy = pymb_rv_policy_share_ownership; + break; + case return_value_policy::take_ownership: + case return_value_policy::copy: + case return_value_policy::move: cap.policy = (pymb_rv_policy) (uint8_t) policy; break; } + + void *result_v = nullptr; if (srcs.downcast.type) { cap.src = srcs.downcast.obj; - if (void *result = try_foreign_bindings(srcs.downcast.type, attempt, &cap)) { - return (PyObject *) result; - } + result_v = try_foreign_bindings(srcs.downcast.type, attempt, &cap); } - cap.src = srcs.original.obj; - if (void *result = try_foreign_bindings(srcs.original.type, attempt, &cap)) { - return (PyObject *) result; + if (!result_v && srcs.original.type) { + cap.src = srcs.original.obj; + result_v = try_foreign_bindings(srcs.original.type, attempt, &cap); } - return nullptr; + + PyObject *result = (PyObject *) result_v; + if (result && policy == return_value_policy::reference_internal && srcs.is_new && + !srcs.used_foreign->keep_alive(result, parent.ptr(), nullptr)) { + keep_alive_impl(result, parent.ptr()); + } + return result; } PYBIND11_NOINLINE static handle cast(const cast_sources &srcs, @@ -1052,8 +1087,8 @@ class type_caster_generic { if (!srcs.result.second) { // No pybind11 type info. See if we can use another framework's // type to complete this cast. Set srcs.used_foreign if so. - if (get_interop_internals().imported_any) { - if (handle ret = cast_foreign(srcs, policy, parent)) { + if (srcs.original.type && get_interop_internals().imported_any) { + if (handle ret = cast_foreign(srcs, policy, parent, existing_holder != nullptr)) { return ret; } } @@ -1143,7 +1178,8 @@ class type_caster_generic { throw cast_error("unhandled return_value_policy: should not happen!"); } - tinfo->init_instance(wrapper, existing_holder); + srcs.init_instance(wrapper, existing_holder); + srcs.is_new = true; return inst.release(); } @@ -1264,14 +1300,14 @@ class type_caster_generic { /// Try to load with foreign typeinfo, if available. Used when there is no /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign(handle src, bool convert) { + PYBIND11_NOINLINE bool try_load_foreign(handle src, bool convert, bool foreign_ok) { constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; const auto pytype = type::handle_of(src); if (hasattr(pytype, local_key)) { return try_load_other_module_local( src, reinterpret_borrow(getattr(pytype, local_key))); } else { - return try_load_other_framework(src, convert); + return foreign_ok && try_load_other_framework(src, convert); } } @@ -1279,13 +1315,13 @@ class type_caster_generic { // bits of code between here and copyable_holder_caster where the two classes need different // logic (without having to resort to virtual inheritance). template - PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { + PYBIND11_NOINLINE bool load_impl(handle src, bool convert, bool foreign_ok = true) { auto &this_ = static_cast(*this); if (!src) { return false; } if (!typeinfo) { - return try_load_foreign(src, convert) && this_.set_foreign_holder(src); + return try_load_foreign(src, convert, foreign_ok) && this_.set_foreign_holder(src); } this_.check_holder_compat(); @@ -1353,14 +1389,14 @@ class type_caster_generic { if (typeinfo->module_local) { if (auto *gtype = get_global_type_info(*typeinfo->cpptype)) { typeinfo = gtype; - return load(src, false); + return load_impl(src, false); } } // Global typeinfo has precedence over foreign module_local and // foreign frameworks - if (try_load_foreign(src, convert)) { - return true; + if (try_load_foreign(src, convert, foreign_ok)) { + return this_.set_foreign_holder(src); } // Custom converters didn't take None, now we convert None to nullptr. @@ -1374,7 +1410,7 @@ class type_caster_generic { } if (convert && cpptype && this_.try_cpp_conduit(src)) { - return true; + return this_.set_foreign_holder(src); } return false; @@ -1724,31 +1760,43 @@ class type_caster_base : public type_caster_generic { make_move_constructor((const itype *) nullptr)); } - template - static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { - handle ret = type_caster_generic::cast( - srcs, return_value_policy::take_ownership, {}, nullptr, nullptr, holder); - if (srcs.used_foreign) { - // Foreign cast succeeded; release C++ ownership - (void) holder->release(); // NOLINT(bugprone-unused-return-value) - } - return ret; - } - PYBIND11_NOINLINE static void after_shared_ptr_cast_to_foreign( handle ret, std::shared_ptr holder, pymb_framework *framework) { // Make the resulting Python object keep a shared_ptr alive, // even if there's not space for it inside the object. std::unique_ptr> sp{new auto(std::move(holder))}; - if (-1 == framework->keep_alive(ret.ptr(), sp.get(), [](void *p) noexcept { - delete (std::shared_ptr *) p; - })) { - ret.dec_ref(); - throw error_already_set(); + auto deleter = [](void *p) noexcept { + delete (std::shared_ptr *) p; + }; + if (!framework->keep_alive(ret.ptr(), sp.get(), deleter)) { + // If they don't provide a keep_alive, use our own weakref-based + // one. If ret is not weakrefable, it will throw and the capsule's + // destructor will clean up for us. + keep_alive_impl(ret, capsule((void *) sp.release(), deleter)); } (void) sp.release(); // NOLINT(bugprone-unused-return-value) } + template + static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { + handle ret = type_caster_generic::cast( + srcs, return_value_policy::take_ownership, {}, nullptr, nullptr, holder); + if (srcs.used_foreign) { + // Foreign cast succeeded; transfer ownership + if (srcs.is_new) { + // The new instance took ownership, so drop it on the C++ side + (void) holder->release(); // NOLINT(bugprone-unused-return-value) + } else { + // The instance already existed, so wouldn't have been carrying + // ownership since `*holder` held exclusive ownership until a + // moment ago. Transfer the ownership via a keep_alive instead. + after_shared_ptr_cast_to_foreign( + ret, std::shared_ptr{std::move(*holder)}, srcs.used_foreign); + } + } + return ret; + } + template static handle cast_holder(const cast_sources &srcs, const std::shared_ptr *holder) { // Use reference policy if casting via a foreign binding, and @@ -1756,13 +1804,20 @@ class type_caster_base : public type_caster_generic { auto policy = srcs.needs_foreign() ? return_value_policy::reference : return_value_policy::take_ownership; handle ret = type_caster_generic::cast(srcs, policy, {}, nullptr, nullptr, holder); - if (srcs.used_foreign) { + if (srcs.used_foreign && srcs.is_new) { after_shared_ptr_cast_to_foreign( ret, std::static_pointer_cast(*holder), srcs.used_foreign); } return ret; } + // Support unique_ptr with custom deleter by converting it to shared_ptr + template + static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { + std::shared_ptr shared_holder = std::move(*holder); + return cast_holder(srcs, &shared_holder); + } + static handle cast_holder(const cast_sources &srcs, const void *holder) { auto policy = srcs.needs_foreign() ? return_value_policy::reference : return_value_policy::take_ownership; diff --git a/include/pybind11/native_enum.h b/include/pybind11/native_enum.h index 5537218f21..80d46afcf3 100644 --- a/include/pybind11/native_enum.h +++ b/include/pybind11/native_enum.h @@ -22,12 +22,12 @@ class native_enum : public detail::native_enum_data { public: using Underlying = typename std::underlying_type::type; - native_enum(const object &parent_scope, + native_enum(handle parent_scope, const char *name, const char *native_type_name, const char *class_doc = "") : detail::native_enum_data( - parent_scope, name, native_type_name, class_doc, std::type_index(typeid(EnumType))) { + parent_scope, name, native_type_name, class_doc, make_info()) { if (detail::get_local_type_info(typeid(EnumType)) != nullptr || detail::get_global_type_info(typeid(EnumType)) != nullptr) { pybind11_fail( @@ -62,6 +62,15 @@ class native_enum : public detail::native_enum_data { native_enum(const native_enum &) = delete; native_enum &operator=(const native_enum &) = delete; + + private: + static detail::native_enum_info make_info() { + detail::native_enum_info ret; + ret.cpptype = &typeid(EnumType); + ret.size_bytes = sizeof(EnumType); + ret.is_signed = std::is_signed::value; + return ret; + } }; PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 351b1b209f..9a0da030c2 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1650,7 +1650,8 @@ class generic_type : public object { auto &interop_internals = get_interop_internals(); if (interop_internals.export_all) { - interop_internals.export_for_interop(tinfo); + interop_internals.export_for_interop( + rec.type, (PyTypeObject *) m_ptr, tinfo); } }); @@ -2926,22 +2927,18 @@ PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { return; /* Nothing to keep alive or nothing to be kept alive by */ } - auto tinfo = all_type_info(Py_TYPE(nurse.ptr())); + PyTypeObject *type = Py_TYPE(nurse.ptr()); + auto tinfo = all_type_info(type); if (!tinfo.empty()) { /* It's a pybind-registered type, so we can store the patient in the * internal list. */ add_patient(nurse.ptr(), patient.ptr()); } else { - if (Py_TYPE(nurse.ptr())->tp_weaklistoffset == 0) { - // The nurse type is not weak-referenceable. Maybe it is a - // different framework's type; try to get them to do the keep_alive. - if (auto *binding = pymb_get_binding(type::handle_of(nurse).ptr())) { - if (0 != binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr)) { - throw error_already_set(); - } - } - // Otherwise continue with the logic below (which will - // raise an error). + auto *binding = pymb_get_binding((PyObject *) type); + if (binding && binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr)) { + // It's a foreign-registered type and the foreign framework was + // able to handle the keep_alive. + return; } /* Fall back to clever approach based on weak references taken from * Boost.Python. This is not used for pybind-registered types because diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 34b22a57ae..801aabc994 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -150,6 +150,7 @@ set(PYBIND11_TEST_FILES test_exceptions test_factory_constructors test_gil_scoped + test_interop.py test_iostream test_kwargs_and_defaults test_local_bindings @@ -247,6 +248,7 @@ tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils") tests_extra_targets("test_cpp_conduit.py" "exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler") +tests_extra_targets("test_interop.py" "test_interop_1;test_interop_2;test_interop_3") set(PYBIND11_EIGEN_REPO "https://gitlab.com/libeigen/eigen.git" diff --git a/tests/test_interop.h b/tests/test_interop.h new file mode 100644 index 0000000000..76f00becd9 --- /dev/null +++ b/tests/test_interop.h @@ -0,0 +1,97 @@ +#pragma once + +#include + +#include +#include + +struct PYBIND11_EXPORT_EXCEPTION SharedExc { + int value; +}; + +struct Shared { + int value; + + enum class Enum { One = 1, Two = 2 }; + + struct Stats { + std::atomic constructed{0}; + std::atomic copied{0}; + std::atomic moved{0}; + std::atomic destroyed{0}; + }; + static Stats& stats() { + static Stats st; + return st; + } + + Shared(int v = 0) : value(v) { ++stats().constructed; } + Shared(const Shared& other) : value(other.value) { ++stats().copied; } + Shared(Shared&& other) noexcept : value(other.value) { ++stats().moved; } + ~Shared() { ++stats().destroyed; } + + static Shared make(int v) { return Shared{v}; } + static std::shared_ptr make_sp(int v) { return std::make_shared(v); } + static std::unique_ptr make_up(int v) { return std::unique_ptr(new Shared{v}); } + static Enum make_enum(int v) { return Enum(v); } + + static int check(const Shared& s) { return s.value; } + static int check_sp(std::shared_ptr s) { return s->value; } + static int check_up(std::unique_ptr s) { return s->value; } + static int check_enum(Enum e) { return (int) e; } + + static long uses(const std::shared_ptr& s) { return s.use_count(); } + + static pybind11::dict pull_stats() { + pybind11::dict ret; + auto& st = stats(); + ret["construct"] = st.constructed.exchange(0); + ret["copy"] = st.copied.exchange(0); + ret["move"] = st.moved.exchange(0); + ret["destroy"] = st.destroyed.exchange(0); + return ret; + } + + template + static void bind_funcs(pybind11::module_ m) { + m.def("make", &make); + m.def("make_sp", &make_sp); + if (SmartHolder) { + m.def("make_up", &make_up); + } else { + // non-smart holder can't bind a unique_ptr return when the + // holder type is shared_ptr + m.def("make_up", [](int v) { return make_up(v).release(); }, + pybind11::return_value_policy::take_ownership); + } + m.def("make_enum", &make_enum); + m.def("check", &check); + m.def("check_sp", &check_sp); + m.def("check_up", &check_up); + m.def("check_enum", &check_enum); + m.def("uses", &uses); + m.def("pull_stats", &pull_stats); + + m.def("export_all", []() { pybind11::interoperate_by_default(true, false); }); + m.def("import_all", []() { pybind11::interoperate_by_default(false, true); }); + m.def("export_for_interop", &pybind11::export_for_interop); + m.def("import_for_interop", &pybind11::import_for_interop<>); + m.def("import_for_interop_explicit", &pybind11::import_for_interop); + struct Other {}; + m.def("import_for_interop_wrong_type", &pybind11::import_for_interop); + } + + template + static void bind_types(pybind11::handle scope) { + using Holder = typename std::conditional>::type; + pybind11::class_(scope, "Shared") + .def_readonly("value", &Shared::value); + pybind11::native_enum(scope, "SharedEnum", "enum.Enum") + .value("One", Enum::One) + .value("Two", Enum::Two) + .finalize(); + pybind11::delattr(scope.attr("Shared"), "_pybind11_conduit_v1_"); + } +}; diff --git a/tests/test_interop.py b/tests/test_interop.py new file mode 100644 index 0000000000..57f99fc4e1 --- /dev/null +++ b/tests/test_interop.py @@ -0,0 +1,564 @@ +# Copyright (c) 2025 The pybind Community. + +import collections +import itertools +import gc +import sys +import time +import threading +import weakref + +import pytest + +import test_interop_1 as t1 +import test_interop_2 as t2 +import test_interop_3 as t3 + +free_threaded = hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() + +# t1, t2, t3 all define bindings for the same C++ type `Shared`, as well as +# several functions to create and inspect instances of that type. Each module +# uses a different PYBIND11_INTERNALS_VERSION, so they won't interoperate +# natively; these tests check the foreign-framework interoperability mechanism. +# The bindings are all different, as follows: +# t1.Shared uses a std::shared_ptr holder +# t2.Shared uses a smart_holder +# t3.RawShared uses the Python C API to mimic a non-pybind11 framework +# Upon import, bindings are defined for the functions, but not for the +# types (`Shared` and `SharedEnum`) until you call bind_types(). +# +# NB: there is some potential for different test cases to interfere with +# each other: we can't un-register a framework once it's registered and we +# can't undo automatic import/export all once they're requested. The ordering +# of these tests is therefore important. They work standalone and they work +# when run all in one process, but they might not work in a different order. + + +def delattr_and_ensure_destroyed(*specs): + wrs = [] + for (mod, name) in specs: + wrs.append(weakref.ref(getattr(mod, name))) + delattr(mod, name) + + for attempt in range(5): + gc.collect() + if all(wr() is None for wr in wrs): + break + else: + pytest.fail( + "Could not delete bindings such as {!r}".format( + next(wr for wr in wrs if wr() is not None) + ) + ) + + + +@pytest.fixture(autouse=True) +def clean_after(): + yield + t3.clear_interop_bindings() + + delattr_and_ensure_destroyed( + *[ + (mod, name) + for mod in (t1, t2, t3) + for name in ("Shared", "SharedEnum", "RawShared") + if hasattr(mod, name) + ] + ) + + t1.pull_stats() + t2.pull_stats() + t3.pull_stats() + + +def check_stats(mod, **entries): + if mod is None: + return + if sys.implementation.name == "pypy": + gc.collect() + stats = mod.pull_stats() + for name, value in entries.items(): + assert stats.pop(name) == value + assert all(val == 0 for val in stats.values()) + + +def test01_interop_exceptions_without_registration(): + # t2 defines the exception translator for Shared. Since it hasn't + # taken any interop actions yet, it hasn't registered with pymetabind + # and t1 won't be able to use that translator. + with pytest.raises(RuntimeError, match="Caught an unknown exception"): + t1.throw_shared(100) + + with pytest.raises(ValueError, match="Shared.200"): + t2.throw_shared(200) + + +global_counter = itertools.count() + + +def expect(from_mod, to_mod, pattern, **extra): + outcomes = {} + extra_info = {} + owner_mod = None + + for idx, suffix in enumerate(("", "_sp", "_up", "_enum")): + create = getattr(from_mod, f"make{suffix}") + check = getattr(to_mod, f"check{suffix}") + thing = suffix.lstrip("_") or "value" + print(thing) + value = idx * 1000 + next(global_counter) + if thing == "enum": + value = (value % 2) + 1 + try: + obj = create(value) + except Exception as ex: + outcomes[thing] = None + extra_info[thing] = ex + continue + if owner_mod is None: + owner_mod = sys.modules[type(obj).__module__] + try: + roundtripped = check(obj) + except Exception as ex: + outcomes[thing] = False + extra_info[thing] = ex + continue + assert roundtripped == value, "instance appears corrupted" + if thing == "sp": + # Include shared_ptr use count in the test. Foreign should create + # a new control block so we see use_count == 1. Local should reuse + # the same -> use_count == 0. + outcomes[thing] = to_mod.uses(obj) + else: + outcomes[thing] = True + + expected = {} + if pattern == "local": + # Accepting a unique_ptr argument only works for non-foreign smart_holder + expected = {"value": True, "sp": 2, "up": to_mod is t2, "enum": True} + elif pattern == "foreign": + expected = {"value": True, "sp": 1, "up": False, "enum": True} + elif pattern == "isolated": + expected = {"value": False, "sp": False, "up": False, "enum": False} + elif pattern == "none": + expected = {"value": None, "sp": None, "up": None, "enum": None} + else: + assert False, "unknown pattern" + expected.update(extra) + assert outcomes == expected + + obj = None + + # When returning by value, we have a construction in from_mod, + # move to owner_mod, destruction in from_mod (after make() returns) + # and destruction in owner mod (when the pyobject dies). + # + # When returning shared_ptr, the construction and destruction both + # occur in from_mod since shared_ptr's deleter is set at creation time. + # + # When returning unique_ptr, the construction occurs in from_mod and + # destruction (when the pyobject dies) occurs in owner_mod; unless + # we pass ownership to to_mod, in which case the destruction happens there. + # But since we can't pass ownership to a foreign framework currently, + # we'll disregard that possibility and always attribute it to owner_mod. + expect_stats = {mod: collections.Counter() for mod in (from_mod, to_mod, owner_mod)} + expect_stats[from_mod].update( + ["construct", "destroy", "construct", "destroy", "construct"] + ) + # value move+destroy + expect_stats[owner_mod].update(["move", "destroy"]) + # unique_ptr destroy; due to an existing pybind11 bug this may be skipped + # entirely (leaked) if we return a raw pointer with rvp take_ownership and + # the cast fails + if owner_mod is None and from_mod is t1: + pass + else: + expect_stats[owner_mod or from_mod].update(["destroy"]) + for mod, stats in expect_stats.items(): + check_stats(mod, **stats) + + +def test02_interop_unimported(): + # Before any types are bound, no to-Python conversions are possible + for mod in (t1, t2, t3): + expect(mod, mod, "none") + + # Bind the types but don't share them yet + t1.bind_types() + t2.bind_types() + + for mod in (t1, t2): + expect(mod, mod, "local") + + # t3 hasn't defined SharedEnum yet. Its version of Shared is not + # bound using pybind11, so is foreign even to the functions in t3. + t3.create_raw_binding() + expect(t3, t3, "foreign", enum=None) + + expect(t1, t2, "isolated") + expect(t1, t3, "isolated") + expect(t2, t1, "isolated") + expect(t2, t3, "isolated") + expect(t3, t1, "isolated", enum=None) + expect(t3, t2, "isolated", enum=None) + + # Just an export isn't enough; you need an import too + t2.export_for_interop(t2.Shared) + expect(t2, t3, "isolated") + + +def test03_interop_import_export_errors(): + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + + with pytest.raises( + RuntimeError, match="type does not define a __pymetabind_binding__" + ): + t2.import_for_interop(t1.Convertible) + + with pytest.raises(RuntimeError, match="not a pybind11 class or enum"): + t3.export_for_interop(t2.Shared) + + with pytest.raises(RuntimeError, match="not a pybind11 class or enum"): + t3.export_for_interop(t2.SharedEnum) + + t2.export_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t2.export_for_interop(t2.Shared) # should be idempotent + t2.export_for_interop(t2.SharedEnum) + + with pytest.raises(RuntimeError, match="type is not foreign"): + t2.import_for_interop(t2.Shared) + + with pytest.raises(RuntimeError, match=r"is not written in C\+\+"): + t2.import_for_interop(t3.RawShared) + + t2.import_for_interop_explicit(t3.RawShared) + t2.import_for_interop_explicit(t3.RawShared) # should be idempotent + + with pytest.raises( + RuntimeError, match=r"was already imported as a different C\+\+ type" + ): + t2.import_for_interop_wrong_type(t3.RawShared) + + +def test04_interop_exceptions(): + # Once t1 and t2 have registered with pymetabind, which happens as soon as + # they each import or export anything, t1 can translate t2's exceptions. + t1.bind_types() + t2.bind_types() + t1.export_for_interop(t1.Shared) + t2.export_for_interop(t2.Shared) + with pytest.raises(ValueError, match="Shared.123"): + t1.throw_shared(123) + + +def test05_interop_with_cpp(): + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + + # Export t1/t2's Shared to t3, but not the enum yet, and not from t3 + t1.export_for_interop(t1.Shared) + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t2.Shared) + expect(t1, t3, "foreign", enum=False) + expect(t2, t3, "foreign", enum=False) + expect(t1, t2, "isolated") + expect(t3, t1, "isolated", enum=None) + expect(t3, t2, "isolated", enum=None) + + # Now export t2.SharedEnum too. Note that t3 doesn't have its own + # definition of SharedEnum yet, so it will use the imported one and create + # t2.SharedEnums. + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + expect(t1, t3, "foreign", enum=False) + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + t1.export_for_interop(t1.SharedEnum) + t3.import_for_interop(t1.SharedEnum) + expect(t1, t3, "foreign") + expect(t2, t1, "isolated") # t1 hasn't imported anything + expect(t2, t3, "foreign") + expect(t3, t1, "isolated") # t3 sends t2.SharedEnums which t1 can't read + + t1.import_for_interop(t2.SharedEnum) + expect(t2, t1, "isolated", enum=True) + expect(t3, t1, "isolated", enum=True) + + t1.import_for_interop(t2.Shared) + expect(t2, t1, "foreign") + + # No one has imported t3.RawShared, so t3->X doesn't work yet + + # t1/t2 each return their local type since it exists (local is always + # preferred). t3 returns its non-pybind extension type because it has + # an import for that one before any of the imports we wrote. + assert type(t1.make(1)) is t1.Shared + assert type(t2.make(2)) is t2.Shared + assert type(t3.make(3)) is t3.RawShared + + # If we create a pybind11 Shared in t3, that takes priority over the raw one + t3.bind_types() + assert type(t3.make(4)) is t3.Shared + + +def test06_interop_return_foreign_smart_holder(): + # Test a pybind11 domain returning a different pybind11 domain's type + # because it didn't have its own. + t2.bind_types() + t2.export_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t1.import_for_interop(t2.Shared) + t1.import_for_interop(t2.SharedEnum) + expect(t2, t1, "foreign") + expect(t1, t2, "local") # t2 is both source and dest + assert type(t1.make(1)) is t2.Shared + assert type(t2.make(2)) is t2.Shared + + +def test06_interop_return_foreign_shared_ptr(): + t1.bind_types() + t1.export_for_interop(t1.Shared) + t1.export_for_interop(t1.SharedEnum) + t2.import_for_interop(t1.Shared) + t2.import_for_interop(t1.SharedEnum) + expect(t1, t2, "foreign") + expect(t2, t1, "local") # t1 is both source and dest + assert type(t1.make(1)) is t1.Shared + assert type(t2.make(2)) is t1.Shared + + +def test07_interop_with_c(): + t1.bind_types() + t3.create_raw_binding() + t1.export_for_interop(t1.SharedEnum) + t3.import_for_interop(t1.SharedEnum) + t1.import_for_interop_explicit(t3.RawShared) + + # Now that t3.RawShared is imported to t1, we can send in the t3->t1 direction. + expect(t3, t1, "foreign") + + +def test08_remove_binding(): + t3.create_raw_binding() + t2.import_for_interop_explicit(t3.RawShared) + + # Remove the binding for t3.RawShared. We expect the t2 domain will + # notice the removal and automatically forget about the defunct binding. + delattr_and_ensure_destroyed((t3, "RawShared")) + t3.create_raw_binding() + + t2.bind_types() + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Similarly test removal of t2.Shared / t2.SharedEnum. + delattr_and_ensure_destroyed((t2, "Shared"), (t2, "SharedEnum")) + t2.bind_types() + + expect(t2, t3, "isolated") + expect(t3, t2, "isolated", enum=None) + + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Removing the binding capsule should work just as well as removing + # the type object. + del t2.Shared.__pymetabind_binding__ + del t2.SharedEnum.__pymetabind_binding__ + gc.collect() + + expect(t2, t3, "isolated") + expect(t3, t2, "isolated", enum=None) + + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Re-import RawShared and now everything works again. + t2.import_for_interop_explicit(t3.RawShared) + expect(t2, t3, "foreign") + expect(t3, t2, "foreign") + + # Removing the binding capsule should work just as well as removing + # the type object. + del t3.RawShared.__pymetabind_binding__ + gc.collect() + t3.export_raw_binding() + + # t3.RawShared was removed from the beginning of t3's list for Shared + # and re-added on the end; also remove and re-add t2.Shared so that + # t3.make() continues to return a t3.RawShared + del t2.Shared.__pymetabind_binding__ + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Re-import RawShared and now everything works again. + t2.import_for_interop_explicit(t3.RawShared) + expect(t2, t3, "foreign") + expect(t3, t2, "foreign") + + +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +def test09_access_binding_concurrently(): + any_failed = False + t3.create_raw_binding() + + def repeatedly_attempt_conversions(): + deadline = time.time() + 1 + while time.time() < deadline: + try: + assert t3.check(t3.make(5)) == 5 + except: + nonlocal any_failed + any_failed = True + raise + + threads = [ + threading.Thread(target=repeatedly_attempt_conversions) for i in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + assert not any_failed + + +@pytest.mark.skipif(not free_threaded, reason="not relevant on non-FT") +def test10_remove_binding_concurrently(): + transitions = 0 + limit = 5000 + + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + + def repeatedly_remove_and_readd(): + nonlocal transitions + try: + while transitions < limit: + del t3.RawShared.__pymetabind_binding__ + t3.export_raw_binding() + transitions += 1 + except: + transitions = limit + raise + + thread = threading.Thread(target=repeatedly_remove_and_readd) + thread.start() + + num_failed = 0 + num_successful = 0 + + def repeatedly_attempt_conversions(): + nonlocal num_failed + nonlocal num_successful + while transitions < limit: + try: + assert t3.check(t3.make(42)) == 42 + except TypeError: + num_failed += 1 + else: + num_successful += 1 + + try: + threads = [ + threading.Thread(target=repeatedly_attempt_conversions) for i in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + finally: + transitions = limit + thread.join() + + # typical numbers from my machine: with limit=5000, + # num_failed and num_successful are each several 10k's + print(num_failed, num_successful) + assert num_successful > 0 + assert num_failed > 0 + + +def test11_implicit(): + # Create four different types of pyobject, all of which have C++ type Shared + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + s1 = t1.make(10) + s2 = t2.make(11) + s3r = t3.make(12) + t3.bind_types() + s3p = t3.make(13) + + t2.export_all() + t3.export_all() + t1.import_all() + t1.import_for_interop_explicit(t3.RawShared) + + assert type(s1) is t1.Shared + assert type(s2) is t2.Shared + assert type(s3r) is t3.RawShared + assert type(s3p) is t3.Shared + + # Test implicit conversions from foreign types + for idx, obj in enumerate((s1, s2, s3r, s3p)): + val = t1.test_implicit(obj) + assert val == 10 + idx + + # We should only be sharing in the tX->t1 direction, not vice versa + assert t1.check(s2) == 11 + with pytest.raises(TypeError): + t2.check(s1) + + # Now add the other direction + t1.export_all() + t2.import_all() + assert t2.check(s1) == 10 + + +def test12_import_export_all(): + # Enable automatic import and export in the t1/t2 domains. + # Still doesn't help with t3->t1/t2 since t3.RawShared is not a C++ type. + t1.import_all() + t1.export_all() + t1.bind_types() + + t2.import_all() + t2.bind_types() + t2.export_all() + + t3.create_raw_binding() + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t2.SharedEnum) + + expect(t1, t2, "foreign") + expect(t1, t3, "foreign", enum=False) # t3 didn't import all or import t1's enum + expect(t2, t1, "foreign") + expect(t2, t3, "isolated", enum=True) # t3 didn't import t2.Shared + t3.import_all() + expect(t2, t3, "foreign") # t3 didn't import t2.Shared + expect(t3, t1, "isolated", enum=True) diff --git a/tests/test_interop_1.cpp b/tests/test_interop_1.cpp new file mode 100644 index 0000000000..1296a0fc4f --- /dev/null +++ b/tests/test_interop_1.cpp @@ -0,0 +1,35 @@ +/* + tests/test_interop_1.cpp -- cross-framework interoperability tests + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Use an unrealistically large internals version to isolate the test_interop +// modules from each other and from the rest of the pybind11 tests +#define PYBIND11_INTERNALS_VERSION 100 + +#include +#include "test_interop.h" + +namespace py = pybind11; + +PYBIND11_MODULE(test_interop_1, m, py::mod_gil_not_used()) { + Shared::bind_funcs(m); + m.def("bind_types", [hm = py::handle(m)]() { + Shared::bind_types(hm); + }); + + m.def("throw_shared", [](int v) { throw SharedExc{v}; }); + + struct Convertible { int value; }; + py::class_(m, "Convertible") + .def(py::init([](const Shared &arg) { + return Convertible{arg.value}; + })) + .def_readonly("value", &Convertible::value); + py::implicitly_convertible(); + m.def("test_implicit", [](Convertible conv) { return conv.value; }); +} diff --git a/tests/test_interop_2.cpp b/tests/test_interop_2.cpp new file mode 100644 index 0000000000..1480297f6b --- /dev/null +++ b/tests/test_interop_2.cpp @@ -0,0 +1,37 @@ +/* + tests/test_interop_2.cpp -- cross-framework interoperability tests + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Use an unrealistically large internals version to isolate the test_interop +// modules from each other and from the rest of the pybind11 tests +#define PYBIND11_INTERNALS_VERSION 200 + +#include +#include "test_interop.h" + +namespace py = pybind11; + +PYBIND11_MODULE(test_interop_2, m, py::mod_gil_not_used()) { + Shared::bind_funcs(m); + m.def("bind_types", [hm = py::handle(m)]() { + Shared::bind_types(hm); + }); + py::register_exception_translator( + [](std::exception_ptr p) { + try { + std::rethrow_exception(p); + } catch (const SharedExc &s) { + // Instead of just calling PyErr_SetString, exercise the + // path where one translator throws an exception to be handled + // by another. + throw py::value_error( + std::string(py::str("Shared({})").format(s.value)).c_str()); + } + }); + m.def("throw_shared", [](int v) { throw SharedExc{v}; }); +} diff --git a/tests/test_interop_3.cpp b/tests/test_interop_3.cpp new file mode 100644 index 0000000000..3eb1a61088 --- /dev/null +++ b/tests/test_interop_3.cpp @@ -0,0 +1,229 @@ +/* + tests/test_interop_3.cpp -- cross-framework interoperability tests + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Use an unrealistically large internals version to isolate the test_interop +// modules from each other and from the rest of the pybind11 tests +#define PYBIND11_INTERNALS_VERSION 300 + +#include +#include +#include "test_interop.h" + +namespace py = pybind11; + +// The following is a manual binding to `struct Shared`, created using the +// CPython C API only. + +struct raw_shared_instance { + PyObject_HEAD + uintptr_t spacer[2]; // ensure instance layout differs from nanobind's + bool destroy; + bool deallocate; + Shared *ptr; + Shared value; + PyObject *weakrefs; +}; + +static void Shared_dealloc(struct raw_shared_instance *self) { + if (self->spacer[0] != 0x5a5a5a5a || self->spacer[1] != 0xa5a5a5a5) + std::terminate(); // instance corrupted + if (self->weakrefs) + PyObject_ClearWeakRefs((PyObject *) self); + if (self->destroy) + self->ptr->~Shared(); + if (self->deallocate) + free(self->ptr); + + PyTypeObject *tp = Py_TYPE((PyObject *) self); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyObject *Shared_new(PyTypeObject *type, Shared *value, pymb_rv_policy rvp) { + struct raw_shared_instance *self; + self = PyObject_New(raw_shared_instance, type); + if (self) { + memset((char *) self + sizeof(PyObject), 0, + sizeof(*self) - sizeof(PyObject)); + self->spacer[0] = 0x5a5a5a5a; + self->spacer[1] = 0xa5a5a5a5; + switch (rvp) { + case pymb_rv_policy_take_ownership: + self->ptr = value; + self->deallocate = self->destroy = true; + break; + case pymb_rv_policy_copy: + new(&self->value) Shared(*value); + self->ptr = &self->value; + self->deallocate = false; + self->destroy = true; + break; + case pymb_rv_policy_move: + new(&self->value) Shared(std::move(*value)); + self->ptr = &self->value; + self->deallocate = false; + self->destroy = true; + break; + case pymb_rv_policy_reference: + case pymb_rv_policy_share_ownership: + self->ptr = value; + self->deallocate = self->destroy = false; + break; + default: + std::terminate(); // unhandled rvp + } + } + return (PyObject *) self; +} + +static int Shared_init(struct raw_shared_instance *, PyObject *, PyObject *) { + PyErr_SetString(PyExc_TypeError, "cannot be constructed from Python"); + return -1; +} + +// And a minimal implementation for our "foreign framework" of the pymetabind +// interface, so nanobind can use raw_shared_instances. + +static void *hook_from_python(pymb_binding *binding, + PyObject *pyobj, + uint8_t, + void (*)(void *ctx, PyObject *obj), + void *) noexcept { + if (binding->pytype != Py_TYPE(pyobj)) + return nullptr; + return ((raw_shared_instance *) pyobj)->ptr; +} + +static PyObject *hook_to_python(pymb_binding *binding, + void *cobj, + enum pymb_rv_policy rvp, + pymb_to_python_feedback *feedback) noexcept { + feedback->relocate = 0; + if (rvp == pymb_rv_policy_none) + return nullptr; + feedback->is_new = 1; + return Shared_new(binding->pytype, (Shared *) cobj, rvp); +} + +static void hook_ignore_foreign_binding(pymb_binding *) noexcept {} +static void hook_ignore_foreign_framework(pymb_framework *) noexcept {} + +PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { + static PyMemberDef Shared_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, + offsetof(struct raw_shared_instance, weakrefs), Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}, + }; + static PyType_Slot Shared_slots[] = { + {Py_tp_doc, (void *) "Shared object"}, + {Py_tp_init, (void *) Shared_init}, + {Py_tp_dealloc, (void *) Shared_dealloc}, + {Py_tp_members, (void *) Shared_members}, + {0, nullptr}, + }; + static PyType_Spec Shared_spec = { + /* name */ "test_interop_3.RawShared", + /* basicsize */ sizeof(struct raw_shared_instance), + /* itemsize */ 0, + /* flags */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + /* slots */ Shared_slots, + }; + + static auto *registry = pymb_get_registry(); + if (!registry) + throw py::error_already_set(); + + static auto *fw = new pymb_framework{}; + fw->name = "example framework for pybind11 tests"; + fw->flags = pymb_framework_leak_safe; + fw->abi_lang = pymb_abi_lang_c; + fw->from_python = hook_from_python; + fw->to_python = hook_to_python; + fw->keep_alive = [](PyObject *, void *, void (*)(void *)) noexcept { + return 0; + }; + fw->remove_local_binding = [](pymb_binding *) noexcept {}; + fw->free_local_binding = [](pymb_binding *binding) noexcept { + delete binding; + }; + fw->add_foreign_binding = hook_ignore_foreign_binding; + fw->remove_foreign_binding = hook_ignore_foreign_binding; + fw->add_foreign_framework = hook_ignore_foreign_framework; + fw->remove_foreign_framework = hook_ignore_foreign_framework; + + pymb_add_framework(registry, fw); + int res = Py_AtExit(+[]() { + pymb_remove_framework(fw); + delete fw; + }); + if (res != 0) + throw py::error_already_set(); + + Shared::bind_funcs(m); + m.def("bind_types", [hm = py::handle(m)]() { + Shared::bind_types(hm); + }); + + m.def("export_raw_binding", [hm = py::handle(m)]() { + auto type = hm.attr("RawShared"); + auto *binding = new pymb_binding{}; + binding->framework = fw; + binding->pytype = (PyTypeObject *) type.ptr(); + binding->source_name = "RawShared"; + pymb_add_binding(binding, /* tp_finalize_will_remove */ 0); + py::import_for_interop(type); + }); + + m.def("create_raw_binding", [hm = py::handle(m)]() { + auto *type = (PyTypeObject *) PyType_FromSpec(&Shared_spec); + if (!type) + throw py::error_already_set(); +#if PY_VERSION_HEX < 0x03090000 + // __weaklistoffset__ member wasn't parsed until 3.9 + type->tp_weaklistoffset = offsetof(struct raw_shared_instance, weakrefs); +#endif + hm.attr("RawShared") = py::reinterpret_steal((PyObject *) type); + hm.attr("export_raw_binding")(); + }); + + m.def("clear_interop_bindings", [hm = py::handle(m)]() { + // NB: this is not a general purpose solution; the bindings removed + // here won't be re-added if `import_all` is called + py::list bound; + pymb_lock_registry(registry); + PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { + bound.append(py::reinterpret_borrow((PyObject *) binding->pytype)); + } + pymb_unlock_registry(registry); + for (auto type : bound) { + py::delattr(type, "__pymetabind_binding__"); + } + + bool bindings_removed = false; + for (int i = 0; i < 5; ++i) { + pymb_lock_registry(registry); + bindings_removed = pymb_list_is_empty(®istry->bindings); + pymb_unlock_registry(registry); + if (bindings_removed) { + break; + } + py::module_::import("gc").attr("collect")(); + } + if (!bindings_removed) { + throw std::runtime_error("Could not remove bindings"); + } + + // Restore the ability for our own create_shared() etc to work + // properly, since that's a foreign type relationship too + if (py::hasattr(hm, "RawShared")) { + hm.attr("export_raw_binding")(); + py::import_for_interop(hm.attr("RawShared")); + } + }); +} From 8bd68a355059c641ceb7db9f5d2e2fe10434a5fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Sep 2025 05:46:05 +0000 Subject: [PATCH 14/18] style: pre-commit fixes --- include/pybind11/cast.h | 11 +-- include/pybind11/detail/common.h | 22 ++--- include/pybind11/detail/foreign.h | 107 +++++++++++---------- include/pybind11/detail/native_enum_data.h | 19 ++-- include/pybind11/detail/type_caster_base.h | 27 +++--- include/pybind11/native_enum.h | 5 +- include/pybind11/pybind11.h | 3 +- tests/extra_python_package/test_files.py | 9 +- tests/test_interop.h | 36 +++---- tests/test_interop.py | 13 +-- tests/test_interop_1.cpp | 13 ++- tests/test_interop_2.cpp | 25 +++-- tests/test_interop_3.cpp | 31 +++--- 13 files changed, 157 insertions(+), 164 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index d4f7fd3dbe..0827746991 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -96,7 +96,7 @@ class type_caster_enum_type { type_caster_base legacy_caster; if (legacy_caster.load(src, convert)) { - legacy_ptr = static_cast(legacy_caster); + legacy_ptr = static_cast(legacy_caster); return true; } return false; @@ -106,14 +106,13 @@ class type_caster_enum_type { using cast_op_type = detail::cast_op_type; // NOLINTNEXTLINE(google-explicit-constructor) - operator EnumType *() { - return native_loaded ? &native_value : legacy_ptr; - } + operator EnumType *() { return native_loaded ? &native_value : legacy_ptr; } // NOLINTNEXTLINE(google-explicit-constructor) operator EnumType &() { - return native_loaded ? native_value : - legacy_ptr ? *legacy_ptr : throw reference_cast_error(); + return native_loaded ? native_value + : legacy_ptr ? *legacy_ptr + : throw reference_cast_error(); } private: diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 543c6dcb6d..8cceb023c7 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -276,18 +276,16 @@ // 3.14 Compatibility #if !defined(Py_GIL_DISABLED) -inline bool is_uniquely_referenced(PyObject *obj) { - return Py_REFCNT(obj) == 1; -} +inline bool is_uniquely_referenced(PyObject *obj) { return Py_REFCNT(obj) == 1; } #elif 0x030E0000 <= PY_VERSION_HEX inline bool is_uniquely_referenced(PyObject *obj) { return PyUnstable_Object_IsUniquelyReferenced(obj); } #else // backport for 3.13 inline bool is_uniquely_referenced(PyObject *obj) { - return _Py_IsOwnedByCurrentThread(obj) && - _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 && - _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0; + return _Py_IsOwnedByCurrentThread(obj) + && _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 + && _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0; } #endif @@ -1369,13 +1367,13 @@ constexpr #endif #if defined(PY_BIG_ENDIAN) -# define PYBIND11_BIG_ENDIAN PY_BIG_ENDIAN +# define PYBIND11_BIG_ENDIAN PY_BIG_ENDIAN #else // pypy doesn't define PY_BIG_ENDIAN -# if defined(_MSC_VER) -# define PYBIND11_BIG_ENDIAN 0 // All Windows platforms are little-endian -# else // GCC and Clang define the following macros -# define PYBIND11_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -# endif +# if defined(_MSC_VER) +# define PYBIND11_BIG_ENDIAN 0 // All Windows platforms are little-endian +# else // GCC and Clang define the following macros +# define PYBIND11_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# endif #endif PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index f28f74a5b4..71b5343af7 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -37,8 +37,8 @@ inline bool should_autoimport_foreign(interop_internals &interop_internals, // Determine whether a pybind11 type is module-local from a different module inline bool is_local_to_other_module(type_info *ti) { - return ti->module_local_load != nullptr && - ti->module_local_load != &type_caster_generic::local_load; + return ti->module_local_load != nullptr + && ti->module_local_load != &type_caster_generic::local_load; } // Add the given `binding` to our type maps so that we can use it to satisfy @@ -73,8 +73,8 @@ inline void *interop_cb_from_python(pymb_binding *binding, return nullptr; } try { - auto cap = reinterpret_borrow( - pytype.attr(native_enum_info::attribute_name())); + auto cap + = reinterpret_borrow(pytype.attr(native_enum_info::attribute_name())); auto *info = cap.get_pointer(); auto value = handle(pyobj).attr("value"); uint64_t ival; @@ -83,8 +83,8 @@ inline void *interop_cb_from_python(pymb_binding *binding, } else { ival = cast(value); } - bytes holder{reinterpret_cast(&ival) + - PYBIND11_BIG_ENDIAN * (8 - info->size_bytes), + bytes holder{reinterpret_cast(&ival) + + PYBIND11_BIG_ENDIAN * (8 - info->size_bytes), info->size_bytes}; keep_referenced(keep_referenced_ctx, holder.ptr()); return PyBytes_AsString(holder.ptr()); @@ -124,7 +124,8 @@ inline void *interop_cb_from_python(pymb_binding *binding, type_caster_generic caster{static_cast(binding->context)}; void *ret = nullptr; try { - if (caster.load_impl(pyobj, convert != 0, + if (caster.load_impl(pyobj, + convert != 0, /* foreign_ok */ false)) { ret = caster.value; } @@ -163,15 +164,15 @@ inline void init_instance_unregistered(instance *inst, const void *holder) { // Undo our shenanigans even if init_instance raises an exception struct guard { - value_and_holder& v_h; + value_and_holder &v_h; ~guard() noexcept { v_h.set_instance_registered(false); if (v_h.type->holder_enum_v == holder_enum_t::smart_holder) { v_h.inst->owned = false; auto &h = v_h.holder(); h.vptr_is_using_std_default_delete = true; - h.reset_vptr_deleter_armed_flag( - v_h.type->get_memory_guarded_delete, /* armed_flag */ false); + h.reset_vptr_deleter_armed_flag(v_h.type->get_memory_guarded_delete, + /* armed_flag */ false); } } } guard{v_h}; @@ -183,7 +184,7 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, enum pymb_rv_policy rvp_, pymb_to_python_feedback *feedback) noexcept { feedback->relocate = 0; // we don't support relocation - feedback->is_new = 0; // unless overridden below + feedback->is_new = 0; // unless overridden below if (cobj == nullptr) { return none().release().ptr(); @@ -193,16 +194,25 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, // Native enum type try { handle pytype((PyObject *) binding->pytype); - auto cap = reinterpret_borrow( - pytype.attr(native_enum_info::attribute_name())); + auto cap + = reinterpret_borrow(pytype.attr(native_enum_info::attribute_name())); auto *info = cap.get_pointer(); uint64_t key; switch (info->size_bytes) { - case 1: key = *(uint8_t *) cobj; break; - case 2: key = *(uint16_t *) cobj; break; - case 4: key = *(uint32_t *) cobj; break; - case 8: key = *(uint64_t *) cobj; break; - default: return nullptr; + case 1: + key = *(uint8_t *) cobj; + break; + case 2: + key = *(uint16_t *) cobj; + break; + case 4: + key = *(uint32_t *) cobj; + break; + case 8: + key = *(uint64_t *) cobj; + break; + default: + return nullptr; } if (rvp_ == pymb_rv_policy_take_ownership) ::operator delete(cobj); @@ -216,7 +226,7 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, return pytype(ikey).release().ptr(); } return pytype(key).release().ptr(); - } catch (error_already_set& exc) { + } catch (error_already_set &exc) { exc.restore(); return nullptr; } @@ -293,21 +303,21 @@ inline int interop_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void } auto cb_to_use = cb ? cb : (decltype(cb)) Py_DecRef; bool success = false; - if (v_h.type->holder_enum_v == holder_enum_t::std_shared_ptr && - !v_h.holder_constructed()) { + if (v_h.type->holder_enum_v == holder_enum_t::std_shared_ptr + && !v_h.holder_constructed()) { // Create a shared_ptr whose destruction will perform the action std::shared_ptr owner(payload, cb_to_use); // Use the aliasing constructor to make its get() return the right thing - new (std::addressof(v_h.holder>())) std::shared_ptr( - std::move(owner), v_h.value_ptr()); + new (std::addressof(v_h.holder>())) + std::shared_ptr(std::move(owner), v_h.value_ptr()); v_h.set_holder_constructed(); success = true; - } else if (v_h.type->holder_enum_v == holder_enum_t::smart_holder && - v_h.holder_constructed() && !v_h.inst->owned) { + } else if (v_h.type->holder_enum_v == holder_enum_t::smart_holder + && v_h.holder_constructed() && !v_h.inst->owned) { auto &h = v_h.holder(); auto *gd = v_h.type->get_memory_guarded_delete(h.vptr); if (gd && !gd->armed_flag) { - gd->del_fun = [=](void*) { cb_to_use(payload); }; + gd->del_fun = [=](void *) { cb_to_use(payload); }; gd->use_del_fun = true; gd->armed_flag = true; success = true; @@ -553,9 +563,8 @@ PYBIND11_NOINLINE void import_for_interop(handle pytype, const std::type_info *c // they're already locked auto &internals = get_internals(); auto it = internals.registered_types_py.find(binding->pytype); - if (it != internals.registered_types_py.end() && - it->second.size() == 1 && - is_local_to_other_module(*it->second.begin())) { + if (it != internals.registered_types_py.end() && it->second.size() == 1 + && is_local_to_other_module(*it->second.begin())) { // Allow importing module-local types from other pybind11 modules, // even if they're ABI-compatible with us and thus use the same // pymb_framework. The import is not doing much here; the export @@ -628,14 +637,13 @@ PYBIND11_NOINLINE void interop_enable_import_all() { // type object. `ti` may be nullptr if exporting a native enum. // Caller must hold the internals lock and have already called // interop_internals.initialize_if_needed(). -PYBIND11_NOINLINE void export_for_interop(const std::type_info *cpptype, - PyTypeObject *pytype, - type_info *ti) { +PYBIND11_NOINLINE void +export_for_interop(const std::type_info *cpptype, PyTypeObject *pytype, type_info *ti) { auto &interop_internals = get_interop_internals(); auto range = interop_internals.bindings.equal_range(*cpptype); for (auto it = range.first; it != range.second; ++it) { - if (it->second->framework == interop_internals.self.get() && - it->second->pytype == pytype) { + if (it->second->framework == interop_internals.self.get() + && it->second->pytype == pytype) { return; // already exported } } @@ -678,10 +686,8 @@ PYBIND11_NOINLINE void interop_enable_export_all() { auto cap = reinterpret_borrow( handle(entry.second).attr(native_enum_info::attribute_name())); auto *info = cap.get_pointer(); - detail::export_for_interop(info->cpptype, - (PyTypeObject *) entry.second, - nullptr); - } catch (error_already_set&) { + detail::export_for_interop(info->cpptype, (PyTypeObject *) entry.second, nullptr); + } catch (error_already_set &) { // Ignore native enums without a __pybind11_enum__ capsule; // they might be from an older version of pybind11 } @@ -707,9 +713,9 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, auto it = range.first; for (; it != range.second; ++it) { auto *binding = it->second; - if (binding->framework == interop_internals.self.get() && - (!binding->context || - !is_local_to_other_module((type_info *) binding->context))) { + if (binding->framework == interop_internals.self.get() + && (!binding->context + || !is_local_to_other_module((type_info *) binding->context))) { // Don't try to use our own types, unless they're module-local // to some other module and this is the only way we'd see them. // (The module-local escape hatch is only relevant for @@ -779,23 +785,19 @@ inline void export_for_interop(handle ty) { interop_internals.initialize_if_needed(); detail::type_info *ti = detail::get_type_info((PyTypeObject *) ty.ptr()); if (ti) { - detail::with_internals([&](detail::internals &) { - detail::export_for_interop(ti->cpptype, ti->type, ti); - }); + detail::with_internals( + [&](detail::internals &) { detail::export_for_interop(ti->cpptype, ti->type, ti); }); return; } // Not a class_; maybe it's a native_enum? try { - auto cap = reinterpret_borrow( - ty.attr(detail::native_enum_info::attribute_name())); + auto cap + = reinterpret_borrow(ty.attr(detail::native_enum_info::attribute_name())); auto *info = cap.get_pointer(); bool ours = detail::with_internals([&](detail::internals &internals) { auto it = internals.native_enum_type_map.find(*info->cpptype); - if (it != internals.native_enum_type_map.end() && - it->second == ty.ptr()) { - detail::export_for_interop(info->cpptype, - (PyTypeObject *) ty.ptr(), - nullptr); + if (it != internals.native_enum_type_map.end() && it->second == ty.ptr()) { + detail::export_for_interop(info->cpptype, (PyTypeObject *) ty.ptr(), nullptr); return true; } return false; @@ -803,7 +805,8 @@ inline void export_for_interop(handle ty) { if (ours) { return; } - } catch (error_already_set&) {} + } catch (error_already_set &) { + } pybind11_fail("pybind11::export_for_interop: not a " "pybind11 class or enum bound in this domain"); } diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index 10c1a3e12d..1f78ec462d 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -33,15 +33,16 @@ class native_enum_data { enum_type_index{*enum_info_.cpptype}, parent_scope(parent_scope), enum_name{enum_name}, native_type_name{native_type_name}, class_doc(class_doc), export_values_flag{false}, finalize_needed{false} { - enum_info = capsule(new native_enum_info{enum_info_}, - native_enum_info::attribute_name(), - +[](void *enum_info_) { - auto *info = (native_enum_info *) enum_info_; - with_internals([&](internals &internals) { - internals.native_enum_type_map.erase(*info->cpptype); - }); - delete info; - }); + enum_info = capsule( + new native_enum_info{enum_info_}, + native_enum_info::attribute_name(), + +[](void *enum_info_) { + auto *info = (native_enum_info *) enum_info_; + with_internals([&](internals &internals) { + internals.native_enum_type_map.erase(*info->cpptype); + }); + delete info; + }); } void finalize(); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 21256fe024..d615d20582 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -9,10 +9,10 @@ #pragma once +#include #include #include #include -#include #include "common.h" #include "cpp_conduit.h" @@ -106,9 +106,7 @@ class loader_life_support { } } - static bool can_add_patient() { - return tls_current_frame() != nullptr; - } + static bool can_add_patient() { return tls_current_frame() != nullptr; } const std::unordered_set &list_patients() const { return keep_alive; } }; @@ -1014,11 +1012,10 @@ class type_caster_generic { return cast(srcs, policy, parent, copy_constructor, move_constructor, existing_holder); } - static handle - cast_foreign(const cast_sources &srcs, - return_value_policy policy, - handle parent, - bool has_holder) { + static handle cast_foreign(const cast_sources &srcs, + return_value_policy policy, + handle parent, + bool has_holder) { struct capture { const void *src; pymb_rv_policy policy = pymb_rv_policy_none; @@ -1044,8 +1041,8 @@ class type_caster_generic { break; case return_value_policy::automatic_reference: case return_value_policy::reference: - cap.policy = has_holder ? pymb_rv_policy_share_ownership - : pymb_rv_policy_reference; + cap.policy + = has_holder ? pymb_rv_policy_share_ownership : pymb_rv_policy_reference; break; case return_value_policy::reference_internal: if (!parent) { @@ -1071,8 +1068,8 @@ class type_caster_generic { } PyObject *result = (PyObject *) result_v; - if (result && policy == return_value_policy::reference_internal && srcs.is_new && - !srcs.used_foreign->keep_alive(result, parent.ptr(), nullptr)) { + if (result && policy == return_value_policy::reference_internal && srcs.is_new + && !srcs.used_foreign->keep_alive(result, parent.ptr(), nullptr)) { keep_alive_impl(result, parent.ptr()); } return result; @@ -1765,9 +1762,7 @@ class type_caster_base : public type_caster_generic { // Make the resulting Python object keep a shared_ptr alive, // even if there's not space for it inside the object. std::unique_ptr> sp{new auto(std::move(holder))}; - auto deleter = [](void *p) noexcept { - delete (std::shared_ptr *) p; - }; + auto deleter = [](void *p) noexcept { delete (std::shared_ptr *) p; }; if (!framework->keep_alive(ret.ptr(), sp.get(), deleter)) { // If they don't provide a keep_alive, use our own weakref-based // one. If ret is not weakrefable, it will throw and the capsule's diff --git a/include/pybind11/native_enum.h b/include/pybind11/native_enum.h index 80d46afcf3..72e124c7a8 100644 --- a/include/pybind11/native_enum.h +++ b/include/pybind11/native_enum.h @@ -26,8 +26,7 @@ class native_enum : public detail::native_enum_data { const char *name, const char *native_type_name, const char *class_doc = "") - : detail::native_enum_data( - parent_scope, name, native_type_name, class_doc, make_info()) { + : detail::native_enum_data(parent_scope, name, native_type_name, class_doc, make_info()) { if (detail::get_local_type_info(typeid(EnumType)) != nullptr || detail::get_global_type_info(typeid(EnumType)) != nullptr) { pybind11_fail( @@ -63,7 +62,7 @@ class native_enum : public detail::native_enum_data { native_enum(const native_enum &) = delete; native_enum &operator=(const native_enum &) = delete; - private: +private: static detail::native_enum_info make_info() { detail::native_enum_info ret; ret.cpptype = &typeid(EnumType); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 9a0da030c2..be712df5ac 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1650,8 +1650,7 @@ class generic_type : public object { auto &interop_internals = get_interop_internals(); if (interop_internals.export_all) { - interop_internals.export_for_interop( - rec.type, (PyTypeObject *) m_ptr, tinfo); + interop_internals.export_for_interop(rec.type, (PyTypeObject *) m_ptr, tinfo); } }); diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index cd4512a873..ef4eac4655 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -135,7 +135,14 @@ "share/pkgconfig/__init__.py", } -headers = main_headers | conduit_headers | detail_headers | contrib_headers | eigen_headers | stl_headers +headers = ( + main_headers + | conduit_headers + | detail_headers + | contrib_headers + | eigen_headers + | stl_headers +) generated_files = cmake_files | pkgconfig_files all_files = headers | generated_files | py_files diff --git a/tests/test_interop.h b/tests/test_interop.h index 76f00becd9..6b2ca9699f 100644 --- a/tests/test_interop.h +++ b/tests/test_interop.h @@ -1,9 +1,9 @@ #pragma once -#include - -#include #include +#include + +#include struct PYBIND11_EXPORT_EXCEPTION SharedExc { int value; @@ -20,31 +20,33 @@ struct Shared { std::atomic moved{0}; std::atomic destroyed{0}; }; - static Stats& stats() { + static Stats &stats() { static Stats st; return st; } Shared(int v = 0) : value(v) { ++stats().constructed; } - Shared(const Shared& other) : value(other.value) { ++stats().copied; } - Shared(Shared&& other) noexcept : value(other.value) { ++stats().moved; } + Shared(const Shared &other) : value(other.value) { ++stats().copied; } + Shared(Shared &&other) noexcept : value(other.value) { ++stats().moved; } ~Shared() { ++stats().destroyed; } static Shared make(int v) { return Shared{v}; } static std::shared_ptr make_sp(int v) { return std::make_shared(v); } - static std::unique_ptr make_up(int v) { return std::unique_ptr(new Shared{v}); } + static std::unique_ptr make_up(int v) { + return std::unique_ptr(new Shared{v}); + } static Enum make_enum(int v) { return Enum(v); } - static int check(const Shared& s) { return s.value; } + static int check(const Shared &s) { return s.value; } static int check_sp(std::shared_ptr s) { return s->value; } static int check_up(std::unique_ptr s) { return s->value; } static int check_enum(Enum e) { return (int) e; } - static long uses(const std::shared_ptr& s) { return s.use_count(); } + static long uses(const std::shared_ptr &s) { return s.use_count(); } static pybind11::dict pull_stats() { pybind11::dict ret; - auto& st = stats(); + auto &st = stats(); ret["construct"] = st.constructed.exchange(0); ret["copy"] = st.copied.exchange(0); ret["move"] = st.moved.exchange(0); @@ -61,8 +63,10 @@ struct Shared { } else { // non-smart holder can't bind a unique_ptr return when the // holder type is shared_ptr - m.def("make_up", [](int v) { return make_up(v).release(); }, - pybind11::return_value_policy::take_ownership); + m.def( + "make_up", + [](int v) { return make_up(v).release(); }, + pybind11::return_value_policy::take_ownership); } m.def("make_enum", &make_enum); m.def("check", &check); @@ -83,11 +87,9 @@ struct Shared { template static void bind_types(pybind11::handle scope) { - using Holder = typename std::conditional>::type; - pybind11::class_(scope, "Shared") - .def_readonly("value", &Shared::value); + using Holder = typename std:: + conditional>::type; + pybind11::class_(scope, "Shared").def_readonly("value", &Shared::value); pybind11::native_enum(scope, "SharedEnum", "enum.Enum") .value("One", Enum::One) .value("Two", Enum::Two) diff --git a/tests/test_interop.py b/tests/test_interop.py index 57f99fc4e1..fc88d850b5 100644 --- a/tests/test_interop.py +++ b/tests/test_interop.py @@ -1,15 +1,15 @@ # Copyright (c) 2025 The pybind Community. +from __future__ import annotations import collections -import itertools import gc +import itertools import sys -import time import threading +import time import weakref import pytest - import test_interop_1 as t1 import test_interop_2 as t2 import test_interop_3 as t3 @@ -36,7 +36,7 @@ def delattr_and_ensure_destroyed(*specs): wrs = [] - for (mod, name) in specs: + for mod, name in specs: wrs.append(weakref.ref(getattr(mod, name))) delattr(mod, name) @@ -46,13 +46,10 @@ def delattr_and_ensure_destroyed(*specs): break else: pytest.fail( - "Could not delete bindings such as {!r}".format( - next(wr for wr in wrs if wr() is not None) - ) + f"Could not delete bindings such as {next(wr for wr in wrs if wr() is not None)!r}" ) - @pytest.fixture(autouse=True) def clean_after(): yield diff --git a/tests/test_interop_1.cpp b/tests/test_interop_1.cpp index 1296a0fc4f..5e4cd0071a 100644 --- a/tests/test_interop_1.cpp +++ b/tests/test_interop_1.cpp @@ -12,23 +12,22 @@ #define PYBIND11_INTERNALS_VERSION 100 #include + #include "test_interop.h" namespace py = pybind11; PYBIND11_MODULE(test_interop_1, m, py::mod_gil_not_used()) { Shared::bind_funcs(m); - m.def("bind_types", [hm = py::handle(m)]() { - Shared::bind_types(hm); - }); + m.def("bind_types", [hm = py::handle(m)]() { Shared::bind_types(hm); }); m.def("throw_shared", [](int v) { throw SharedExc{v}; }); - struct Convertible { int value; }; + struct Convertible { + int value; + }; py::class_(m, "Convertible") - .def(py::init([](const Shared &arg) { - return Convertible{arg.value}; - })) + .def(py::init([](const Shared &arg) { return Convertible{arg.value}; })) .def_readonly("value", &Convertible::value); py::implicitly_convertible(); m.def("test_implicit", [](Convertible conv) { return conv.value; }); diff --git a/tests/test_interop_2.cpp b/tests/test_interop_2.cpp index 1480297f6b..f31852748a 100644 --- a/tests/test_interop_2.cpp +++ b/tests/test_interop_2.cpp @@ -12,26 +12,23 @@ #define PYBIND11_INTERNALS_VERSION 200 #include + #include "test_interop.h" namespace py = pybind11; PYBIND11_MODULE(test_interop_2, m, py::mod_gil_not_used()) { Shared::bind_funcs(m); - m.def("bind_types", [hm = py::handle(m)]() { - Shared::bind_types(hm); + m.def("bind_types", [hm = py::handle(m)]() { Shared::bind_types(hm); }); + py::register_exception_translator([](std::exception_ptr p) { + try { + std::rethrow_exception(p); + } catch (const SharedExc &s) { + // Instead of just calling PyErr_SetString, exercise the + // path where one translator throws an exception to be handled + // by another. + throw py::value_error(std::string(py::str("Shared({})").format(s.value)).c_str()); + } }); - py::register_exception_translator( - [](std::exception_ptr p) { - try { - std::rethrow_exception(p); - } catch (const SharedExc &s) { - // Instead of just calling PyErr_SetString, exercise the - // path where one translator throws an exception to be handled - // by another. - throw py::value_error( - std::string(py::str("Shared({})").format(s.value)).c_str()); - } - }); m.def("throw_shared", [](int v) { throw SharedExc{v}; }); } diff --git a/tests/test_interop_3.cpp b/tests/test_interop_3.cpp index 3eb1a61088..4c25631c17 100644 --- a/tests/test_interop_3.cpp +++ b/tests/test_interop_3.cpp @@ -11,8 +11,9 @@ // modules from each other and from the rest of the pybind11 tests #define PYBIND11_INTERNALS_VERSION 300 -#include #include +#include + #include "test_interop.h" namespace py = pybind11; @@ -49,8 +50,7 @@ static PyObject *Shared_new(PyTypeObject *type, Shared *value, pymb_rv_policy rv struct raw_shared_instance *self; self = PyObject_New(raw_shared_instance, type); if (self) { - memset((char *) self + sizeof(PyObject), 0, - sizeof(*self) - sizeof(PyObject)); + memset((char *) self + sizeof(PyObject), 0, sizeof(*self) - sizeof(PyObject)); self->spacer[0] = 0x5a5a5a5a; self->spacer[1] = 0xa5a5a5a5; switch (rvp) { @@ -59,13 +59,13 @@ static PyObject *Shared_new(PyTypeObject *type, Shared *value, pymb_rv_policy rv self->deallocate = self->destroy = true; break; case pymb_rv_policy_copy: - new(&self->value) Shared(*value); + new (&self->value) Shared(*value); self->ptr = &self->value; self->deallocate = false; self->destroy = true; break; case pymb_rv_policy_move: - new(&self->value) Shared(std::move(*value)); + new (&self->value) Shared(std::move(*value)); self->ptr = &self->value; self->deallocate = false; self->destroy = true; @@ -116,8 +116,11 @@ static void hook_ignore_foreign_framework(pymb_framework *) noexcept {} PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { static PyMemberDef Shared_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, - offsetof(struct raw_shared_instance, weakrefs), Py_READONLY, nullptr}, + {"__weaklistoffset__", + Py_T_PYSSIZET, + offsetof(struct raw_shared_instance, weakrefs), + Py_READONLY, + nullptr}, {nullptr, 0, 0, 0, nullptr}, }; static PyType_Slot Shared_slots[] = { @@ -145,13 +148,9 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { fw->abi_lang = pymb_abi_lang_c; fw->from_python = hook_from_python; fw->to_python = hook_to_python; - fw->keep_alive = [](PyObject *, void *, void (*)(void *)) noexcept { - return 0; - }; + fw->keep_alive = [](PyObject *, void *, void (*)(void *)) noexcept { return 0; }; fw->remove_local_binding = [](pymb_binding *) noexcept {}; - fw->free_local_binding = [](pymb_binding *binding) noexcept { - delete binding; - }; + fw->free_local_binding = [](pymb_binding *binding) noexcept { delete binding; }; fw->add_foreign_binding = hook_ignore_foreign_binding; fw->remove_foreign_binding = hook_ignore_foreign_binding; fw->add_foreign_framework = hook_ignore_foreign_framework; @@ -166,9 +165,7 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { throw py::error_already_set(); Shared::bind_funcs(m); - m.def("bind_types", [hm = py::handle(m)]() { - Shared::bind_types(hm); - }); + m.def("bind_types", [hm = py::handle(m)]() { Shared::bind_types(hm); }); m.def("export_raw_binding", [hm = py::handle(m)]() { auto type = hm.attr("RawShared"); @@ -197,7 +194,7 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { // here won't be re-added if `import_all` is called py::list bound; pymb_lock_registry(registry); - PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { + PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { bound.append(py::reinterpret_borrow((PyObject *) binding->pytype)); } pymb_unlock_registry(registry); From d88a14bac7b1d4e7434ea0f43e46c02e5c2cdf48 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 15 Sep 2025 23:54:29 -0600 Subject: [PATCH 15/18] Fix CI --- include/pybind11/detail/foreign.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index 71b5343af7..e62ce8c2a6 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -148,6 +148,7 @@ inline void *interop_cb_from_python(pymb_binding *binding, // can fix up the holder before other threads start using the new instance. inline void init_instance_unregistered(instance *inst, const void *holder) { assert(holder == nullptr && !inst->owned); + (void) holder; // avoid unused warning if compiled without asserts value_and_holder v_h = *values_and_holders(inst).begin(); // If using smart_holder, force creation of a shared_ptr that has a From 0a08801d2103b978e039f52a9ff224d69258bfd9 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 16 Sep 2025 00:07:58 -0600 Subject: [PATCH 16/18] Fix pre-commit issues --- include/pybind11/detail/internals.h | 1 + include/pybind11/detail/type_caster_base.h | 2 +- tests/test_interop.py | 7 +++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index bdeebdee2e..849d1f9d32 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -28,6 +28,7 @@ struct pymb_binding; struct pymb_framework; struct pymb_registry; + /// Tracks the `internals` and `type_info` ABI version independent of the main library version. /// /// Some portions of the code use an ABI that is conditional depending on this diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index d615d20582..598157831f 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -1372,7 +1372,7 @@ class type_caster_generic { if (convert) { for (const auto &converter : typeinfo->implicit_conversions) { auto temp = reinterpret_steal(converter(src.ptr(), typeinfo->type)); - if (load_impl(temp, false)) { + if (load_impl(temp, false)) { // codespell:ignore ThisT loader_life_support::add_patient(temp); return true; } diff --git a/tests/test_interop.py b/tests/test_interop.py index fc88d850b5..cad867408d 100644 --- a/tests/test_interop.py +++ b/tests/test_interop.py @@ -1,5 +1,4 @@ -# Copyright (c) 2025 The pybind Community. -from __future__ import annotations +# Copyright (c) 2025 The pybind from __future__ import annotations import collections import gc @@ -40,7 +39,7 @@ def delattr_and_ensure_destroyed(*specs): wrs.append(weakref.ref(getattr(mod, name))) delattr(mod, name) - for attempt in range(5): + for _ in range(5): gc.collect() if all(wr() is None for wr in wrs): break @@ -141,7 +140,7 @@ def expect(from_mod, to_mod, pattern, **extra): elif pattern == "none": expected = {"value": None, "sp": None, "up": None, "enum": None} else: - assert False, "unknown pattern" + pytest.fail("unknown pattern") expected.update(extra) assert outcomes == expected From 11c71cd5352271867488a67905bf6bd4d0747b01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Sep 2025 06:08:24 +0000 Subject: [PATCH 17/18] style: pre-commit fixes --- include/pybind11/detail/internals.h | 1 - tests/test_interop.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 849d1f9d32..bdeebdee2e 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -28,7 +28,6 @@ struct pymb_binding; struct pymb_framework; struct pymb_registry; - /// Tracks the `internals` and `type_info` ABI version independent of the main library version. /// /// Some portions of the code use an ABI that is conditional depending on this diff --git a/tests/test_interop.py b/tests/test_interop.py index cad867408d..751a80e8e1 100644 --- a/tests/test_interop.py +++ b/tests/test_interop.py @@ -1,4 +1,5 @@ # Copyright (c) 2025 The pybind from __future__ import annotations +from __future__ import annotations import collections import gc From a21e7bcce9fd17e92f3c63aca57aacb25a7c1a43 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 16 Sep 2025 00:16:46 -0600 Subject: [PATCH 18/18] More CI fixes --- .codespell-ignore-lines | 1 + include/pybind11/cast.h | 7 +- include/pybind11/contrib/pymetabind.h | 30 ++-- include/pybind11/detail/common.h | 4 +- include/pybind11/detail/foreign.h | 79 ++++----- include/pybind11/detail/internals.h | 73 ++++++++- include/pybind11/detail/native_enum_data.h | 2 +- include/pybind11/detail/type_caster_base.h | 16 +- include/pybind11/pybind11.h | 2 +- tests/test_interop.h | 12 ++ tests/test_interop.py | 177 ++++++++++++--------- tests/test_interop_1.cpp | 3 +- tests/test_interop_2.cpp | 3 +- tests/test_interop_3.cpp | 86 ++++++---- 14 files changed, 321 insertions(+), 174 deletions(-) diff --git a/.codespell-ignore-lines b/.codespell-ignore-lines index e8cbf31447..c03cd2f727 100644 --- a/.codespell-ignore-lines +++ b/.codespell-ignore-lines @@ -2,6 +2,7 @@ template auto &this_ = static_cast(*this); if (load_impl(temp, false)) { + return load_impl(src, false); ssize_t nd = 0; auto trivial = broadcast(buffers, nd, shape); auto ndim = (size_t) nd; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 0827746991..259866fc1c 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -110,9 +110,10 @@ class type_caster_enum_type { // NOLINTNEXTLINE(google-explicit-constructor) operator EnumType &() { - return native_loaded ? native_value - : legacy_ptr ? *legacy_ptr - : throw reference_cast_error(); + if (!native_loaded && !legacy_ptr) { + throw reference_cast_error(); + } + return native_loaded ? native_value : *legacy_ptr; } private: diff --git a/include/pybind11/contrib/pymetabind.h b/include/pybind11/contrib/pymetabind.h index 1b7c52efe3..53a9186e99 100644 --- a/include/pybind11/contrib/pymetabind.h +++ b/include/pybind11/contrib/pymetabind.h @@ -6,7 +6,10 @@ * This functionality is intended to be used by the framework itself, * rather than by users of the framework. * - * This is version 0.3 of pymetabind. Changelog: + * This is version 0.3+dev of pymetabind. Changelog: + * + * Unreleased: Properly return NULL if registry capsule creation fails. + * Support GraalPy. * * Version 0.3: Don't do a Py_DECREF in `pymb_remove_framework` since the * 2025-09-15 interpreter might already be finalized at that point. @@ -727,6 +730,8 @@ PYMB_FUNC void pymb_remove_framework(struct pymb_framework* framework); PYMB_FUNC void pymb_add_binding(struct pymb_binding* binding, int tp_finalize_will_remove); PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding); +PYMB_FUNC void pymb_remove_binding_internal(struct pymb_binding* binding, + int from_capsule_destructor); PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); #if !defined(PYMB_DECLS_ONLY) @@ -777,7 +782,7 @@ PYMB_FUNC PyObject* pymb_weakref_callback(PyObject* self, PyObject* weakref) { * import lock can provide mutual exclusion. */ PYMB_FUNC struct pymb_registry* pymb_get_registry() { -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) PyObject* dict = PyEval_GetBuiltins(); #elif PY_VERSION_HEX < 0x03090000 PyObject* dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); @@ -822,6 +827,7 @@ PYMB_FUNC struct pymb_registry* pymb_get_registry() { pymb_registry_capsule_destructor); if (!capsule) { free(registry); + registry = NULL; } else if (PyDict_SetItem(dict, key, capsule) == -1) { registry = NULL; // will be deallocated by capsule destructor } @@ -906,7 +912,7 @@ PYMB_FUNC void pymb_binding_capsule_remove(PyObject* capsule) { PyErr_WriteUnraisable(capsule); return; } - pymb_remove_binding(binding); + pymb_remove_binding_internal(binding, /* from_capsule_destructor */ 1); } /* @@ -1010,6 +1016,11 @@ PYMB_FUNC void pymb_binding_capsule_destroy(PyObject* capsule) { * `binding->framework->free_local_binding(binding)`. */ PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding) { + pymb_remove_binding_internal(binding, 0); +} + +PYMB_FUNC void pymb_remove_binding_internal(struct pymb_binding* binding, + int from_capsule_destructor) { struct pymb_registry* registry = binding->framework->registry; // Since we need to obtain it anyway, use the registry lock to serialize @@ -1037,12 +1048,13 @@ PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding) { // Clear the existing capsule's destructor so we don't have to worry about // it firing after the pymb_binding struct has actually been freed. - // Note we can safely assume the capsule hasn't been freed yet, even - // though it might be mid-destruction. (Proof: Its destructor calls - // this function, which cannot complete until it acquires the lock we - // currently hold. If the destructor completed already, we would have bailed - // out above upon noticing capsule was already NULL.) - if (PyCapsule_SetDestructor(binding->capsule, NULL) != 0) { + // Note we can safely assume the capsule hasn't been freed yet. + // (Proof: Its destructor calls this function, which cannot complete until + // it acquires the lock we currently hold. If the destructor completed + // already, we would have bailed out above upon noticing capsule was + // already NULL.) + if (!from_capsule_destructor && + PyCapsule_SetDestructor(binding->capsule, NULL) != 0) { PyErr_WriteUnraisable((PyObject *) binding->pytype); } diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 8cceb023c7..6196b5ad0d 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -284,8 +284,8 @@ inline bool is_uniquely_referenced(PyObject *obj) { #else // backport for 3.13 inline bool is_uniquely_referenced(PyObject *obj) { return _Py_IsOwnedByCurrentThread(obj) - && _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 - && _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0; + && _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local) == 1 + && _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared) == 0; } #endif diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h index e62ce8c2a6..f76aa032c0 100644 --- a/include/pybind11/detail/foreign.h +++ b/include/pybind11/detail/foreign.h @@ -22,8 +22,9 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { auto &interop_internals = get_interop_internals(); for (pymb_framework *fw : interop_internals.exc_frameworks) { - if (fw->translate_exception(&p)) + if (fw->translate_exception(&p) != 0) { return; + } } std::rethrow_exception(p); } @@ -47,14 +48,14 @@ inline void import_foreign_binding(pymb_binding *binding, const std::type_info * // Caller must hold the internals lock auto &interop_internals = get_interop_internals(); interop_internals.imported_any = true; - auto range = interop_internals.bindings.equal_range(*cpptype); - for (auto it = range.first; it != range.second; ++it) { - if (it->second == binding) { + auto &lst = interop_internals.bindings[*cpptype]; + for (pymb_binding *existing : lst) { + if (existing == binding) { return; // already imported } } ++interop_internals.bindings_update_count; - interop_internals.bindings.emplace(*cpptype, binding); + lst.append(binding); } // Callback functions for other frameworks to operate on our objects @@ -77,14 +78,14 @@ inline void *interop_cb_from_python(pymb_binding *binding, = reinterpret_borrow(pytype.attr(native_enum_info::attribute_name())); auto *info = cap.get_pointer(); auto value = handle(pyobj).attr("value"); - uint64_t ival; + uint64_t ival = 0; if (info->is_signed && handle(value) < int_(0)) { ival = (uint64_t) cast(value); } else { ival = cast(value); } bytes holder{reinterpret_cast(&ival) - + PYBIND11_BIG_ENDIAN * (8 - info->size_bytes), + + PYBIND11_BIG_ENDIAN * size_t(8 - info->size_bytes), info->size_bytes}; keep_referenced(keep_referenced_ctx, holder.ptr()); return PyBytes_AsString(holder.ptr()); @@ -198,7 +199,7 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, auto cap = reinterpret_borrow(pytype.attr(native_enum_info::attribute_name())); auto *info = cap.get_pointer(); - uint64_t key; + uint64_t key = 0; switch (info->size_bytes) { case 1: key = *(uint8_t *) cobj; @@ -215,10 +216,11 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, default: return nullptr; } - if (rvp_ == pymb_rv_policy_take_ownership) + if (rvp_ == pymb_rv_policy_take_ownership) { ::operator delete(cobj); + } if (info->is_signed) { - int64_t ikey = (int64_t) key; + auto ikey = (int64_t) key; if (info->size_bytes < 8) { // sign extend ikey <<= (64 - (info->size_bytes * 8)); @@ -275,7 +277,7 @@ inline PyObject *interop_cb_to_python(pymb_binding *binding, srcs.init_instance = init_instance_unregistered; } handle ret = type_caster_generic::cast(srcs, rvp, {}, copy_ctor, move_ctor); - feedback->is_new = srcs.is_new; + feedback->is_new = uint8_t(srcs.is_new); return ret.ptr(); } catch (...) { translate_exception(std::current_exception()); @@ -309,7 +311,9 @@ inline int interop_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void // Create a shared_ptr whose destruction will perform the action std::shared_ptr owner(payload, cb_to_use); // Use the aliasing constructor to make its get() return the right thing + // NB: this constructor accepts an rvalue reference only in C++20 new (std::addressof(v_h.holder>())) + // NOLINTNEXTLINE(performance-move-const-arg) std::shared_ptr(std::move(owner), v_h.value_ptr()); v_h.set_holder_constructed(); success = true; @@ -403,13 +407,12 @@ inline int interop_cb_translate_exception(void *eptr) noexcept { inline void interop_cb_remove_local_binding(pymb_binding *binding) noexcept { with_internals([&](internals &) { auto &interop_internals = get_interop_internals(); - auto *cpptype = (const std::type_info *) binding->native_type; - auto range = interop_internals.bindings.equal_range(*cpptype); - for (auto it = range.first; it != range.second; ++it) { - if (it->second == binding) { - ++interop_internals.bindings_update_count; + const auto *cpptype = (const std::type_info *) binding->native_type; + auto it = interop_internals.bindings.find(*cpptype); + if (it != interop_internals.bindings.end() && it->second.erase(binding)) { + ++interop_internals.bindings_update_count; + if (it->second.empty()) { interop_internals.bindings.erase(it); - return; } } }); @@ -433,12 +436,11 @@ inline void interop_cb_remove_foreign_binding(pymb_binding *binding) noexcept { with_internals([&](internals &) { auto &interop_internals = get_interop_internals(); auto remove_from_type = [&](const std::type_info *type) { - auto range = interop_internals.bindings.equal_range(*type); - for (auto it = range.first; it != range.second; ++it) { - if (it->second == binding) { - ++interop_internals.bindings_update_count; + auto it = interop_internals.bindings.find(*type); + if (it != interop_internals.bindings.end() && it->second.erase(binding)) { + ++interop_internals.bindings_update_count; + if (it->second.empty()) { interop_internals.bindings.erase(it); - break; } } }; @@ -641,11 +643,10 @@ PYBIND11_NOINLINE void interop_enable_import_all() { PYBIND11_NOINLINE void export_for_interop(const std::type_info *cpptype, PyTypeObject *pytype, type_info *ti) { auto &interop_internals = get_interop_internals(); - auto range = interop_internals.bindings.equal_range(*cpptype); - for (auto it = range.first; it != range.second; ++it) { - if (it->second->framework == interop_internals.self.get() - && it->second->pytype == pytype) { - return; // already exported + auto &lst = interop_internals.bindings[*cpptype]; + for (pymb_binding *existing : lst) { + if (existing->framework == interop_internals.self.get() && existing->pytype == pytype) { + return; // already imported } } @@ -657,7 +658,7 @@ export_for_interop(const std::type_info *cpptype, PyTypeObject *pytype, type_inf binding->context = ti; ++interop_internals.bindings_update_count; - interop_internals.bindings.emplace(*cpptype, binding); + lst.append(binding); pymb_add_binding(binding, /* tp_finalize_will_remove */ 0); } @@ -688,7 +689,7 @@ PYBIND11_NOINLINE void interop_enable_export_all() { handle(entry.second).attr(native_enum_info::attribute_name())); auto *info = cap.get_pointer(); detail::export_for_interop(info->cpptype, (PyTypeObject *) entry.second, nullptr); - } catch (error_already_set &) { + } catch (error_already_set &) { // NOLINT(bugprone-empty-catch) // Ignore native enums without a __pybind11_enum__ capsule; // they might be from an older version of pybind11 } @@ -710,10 +711,11 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, do { PYBIND11_LOCK_INTERNALS(internals); (void) internals; // suppress unused warning on non-ft builds - auto range = interop_internals.bindings.equal_range(*type); - auto it = range.first; - for (; it != range.second; ++it) { - auto *binding = it->second; + auto it = interop_internals.bindings.find(*type); + if (it == interop_internals.bindings.end()) { + return nullptr; + } + for (pymb_binding *binding : it->second) { if (binding->framework == interop_internals.self.get() && (!binding->context || !is_local_to_other_module((type_info *) binding->context))) { @@ -741,13 +743,13 @@ PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, // was done within attempt(), or concurrently during attempt() // while we didn't hold the internals lock if (interop_internals.bindings_update_count != update_count) { - // Concurrent update occurred; retry - update_count = interop_internals.bindings_update_count; + // Concurrent update occurred; stop iterating break; } } - if (it != range.second) { + if (interop_internals.bindings_update_count != update_count) { // We broke out early due to a concurrent update. Retry from the top. + update_count = interop_internals.bindings_update_count; continue; } return nullptr; @@ -775,7 +777,7 @@ inline void import_for_interop(handle pytype) { auto &interop_internals = detail::get_interop_internals(); interop_internals.initialize_if_needed(); detail::with_internals( - [&](detail::internals &) { detail::import_for_interop(std::move(pytype), cpptype); }); + [&](detail::internals &) { detail::import_for_interop(pytype, cpptype); }); } inline void export_for_interop(handle ty) { @@ -806,7 +808,8 @@ inline void export_for_interop(handle ty) { if (ours) { return; } - } catch (error_already_set &) { + } catch (error_already_set &) { // NOLINT(bugprone-empty-catch) + // Could be an older native enum without __pybind11_enum__ capsule } pybind11_fail("pybind11::export_for_interop: not a " "pybind11 class or enum bound in this domain"); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 748f0e8aac..92dc6850c7 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -184,9 +184,6 @@ struct type_equal_to { template using type_map = std::unordered_map; -template -using type_multimap - = std::unordered_multimap; struct override_hash { inline size_t operator()(const std::pair &v) const { @@ -313,10 +310,73 @@ using copy_or_move_ctor = void *(*) (const void *); // indicating whether any foreign bindings are also known for its C++ type; // that way we can avoid an extra lookup when conversion to a native type fails. struct interop_internals { + // A vector optimized for the case of storing one element. + class binding_list { + public: + binding_list() : single(nullptr) {} + binding_list(const binding_list &) = delete; + binding_list(binding_list &&other) noexcept : vec(other.vec) { other.vec = 0; } + ~binding_list() { + if (is_vec()) { + delete as_vec(); + } + } + + bool empty() const { return single == nullptr; } + pymb_binding **begin() { + return empty() ? nullptr : is_vec() ? as_vec()->data() : &single; + } + pymb_binding **end() { + return empty() ? nullptr + : is_vec() ? as_vec()->data() + as_vec()->size() + : &single + 1; + } + void append(pymb_binding *binding) { + if (empty()) { + single = binding; + } else if (!is_vec()) { + vec = uintptr_t(new Vec{single, binding}) | uintptr_t(1); + } else { + as_vec()->push_back(binding); + } + } + bool erase(pymb_binding *binding) { + if (is_vec()) { + Vec *v = as_vec(); + for (auto it = v->begin(); it != v->end(); ++it) { + if (*it == binding) { + v->erase(it); + if (v->size() == 1) { + single = (*v)[0]; + delete v; + } + return true; + } + } + } else if (single == binding) { + single = nullptr; + return true; + } + return false; + } + + private: + // Discriminated pointer: points to pymb_binding if the low bit is clear, + // or to std::vector if the low bit is set. + using Vec = std::vector; + union { + pymb_binding *single; + uintptr_t vec; + }; + bool is_vec() const { return (vec & uintptr_t(1)) != 0; } + // NOLINTNEXTLINE(performance-no-int-to-ptr) + Vec *as_vec() const { return (Vec *) (vec & ~uintptr_t(1)); } + }; + // Registered pymetabind bindings for each C++ type, including both our // own (exported) and other frameworks' (imported). // Protected by internals::mutex. - type_multimap bindings; + type_map bindings; // For each C++ type with a native binding, store pointers to its // copy and move constructors. These would ideally move inside `type_info` @@ -801,9 +861,8 @@ inline auto with_exception_translators(const F &cb) } inline internals_pp_manager &get_interop_internals_pp_manager() { - static internals_pp_manager interop_internals_pp_manager( - PYBIND11_INTERNALS_ID "interop", nullptr); - return interop_internals_pp_manager; + return internals_pp_manager::get_instance(PYBIND11_INTERNALS_ID "interop", + nullptr); } inline interop_internals &get_interop_internals() { diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index 1f78ec462d..56cd121e05 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -215,7 +215,7 @@ inline void native_enum_data::finalize() { auto &interop_internals = get_interop_internals(); if (interop_internals.export_all) { - auto info = enum_info.get_pointer(); + auto *info = enum_info.get_pointer(); interop_internals.export_for_interop( info->cpptype, (PyTypeObject *) py_enum.ptr(), nullptr); } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 598157831f..d936cecfe3 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -266,9 +266,10 @@ PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, auto &interop_internals = detail::get_interop_internals(); if (interop_internals.imported_any) { handle ret = with_internals([&](internals &) { - auto range = interop_internals.bindings.equal_range(tp); - if (range.first != range.second) { - return handle((PyObject *) range.first->second->pytype); + auto it = interop_internals.bindings.find(tp); + if (it != interop_internals.bindings.end() && !it->second.empty()) { + pymb_binding *binding = *it->second.begin(); + return handle((PyObject *) binding->pytype); } return handle(); }); @@ -1067,9 +1068,9 @@ class type_caster_generic { result_v = try_foreign_bindings(srcs.original.type, attempt, &cap); } - PyObject *result = (PyObject *) result_v; + auto *result = (PyObject *) result_v; if (result && policy == return_value_policy::reference_internal && srcs.is_new - && !srcs.used_foreign->keep_alive(result, parent.ptr(), nullptr)) { + && srcs.used_foreign->keep_alive(result, parent.ptr(), nullptr) == 0) { keep_alive_impl(result, parent.ptr()); } return result; @@ -1303,9 +1304,8 @@ class type_caster_generic { if (hasattr(pytype, local_key)) { return try_load_other_module_local( src, reinterpret_borrow(getattr(pytype, local_key))); - } else { - return foreign_ok && try_load_other_framework(src, convert); } + return foreign_ok && try_load_other_framework(src, convert); } // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant @@ -1372,7 +1372,7 @@ class type_caster_generic { if (convert) { for (const auto &converter : typeinfo->implicit_conversions) { auto temp = reinterpret_steal(converter(src.ptr(), typeinfo->type)); - if (load_impl(temp, false)) { // codespell:ignore ThisT + if (load_impl(temp, false)) { loader_life_support::add_patient(temp); return true; } diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index be712df5ac..ce51188057 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2934,7 +2934,7 @@ PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { add_patient(nurse.ptr(), patient.ptr()); } else { auto *binding = pymb_get_binding((PyObject *) type); - if (binding && binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr)) { + if (binding && binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr) != 0) { // It's a foreign-registered type and the foreign framework was // able to handle the keep_alive. return; diff --git a/tests/test_interop.h b/tests/test_interop.h index 6b2ca9699f..42af357f5c 100644 --- a/tests/test_interop.h +++ b/tests/test_interop.h @@ -25,6 +25,7 @@ struct Shared { return st; } + // NOLINTNEXTLINE(google-explicit-constructor) Shared(int v = 0) : value(v) { ++stats().constructed; } Shared(const Shared &other) : value(other.value) { ++stats().copied; } Shared(Shared &&other) noexcept : value(other.value) { ++stats().moved; } @@ -38,6 +39,7 @@ struct Shared { static Enum make_enum(int v) { return Enum(v); } static int check(const Shared &s) { return s.value; } + // NOLINTNEXTLINE(performance-unnecessary-value-param) static int check_sp(std::shared_ptr s) { return s->value; } static int check_up(std::unique_ptr s) { return s->value; } static int check_enum(Enum e) { return (int) e; } @@ -87,6 +89,16 @@ struct Shared { template static void bind_types(pybind11::handle scope) { + if (pybind11::hasattr(scope, "Shared")) { + if (pybind11::detail::get_interop_internals().export_all) { + // If bindings were removed but types weren't (because types + // are immortal in this environment) then manually re-export + // the bindings so that the effects of export_all are observable + pybind11::export_for_interop(scope.attr("Shared")); + pybind11::export_for_interop(scope.attr("SharedEnum")); + } + return; + } using Holder = typename std:: conditional>::type; pybind11::class_(scope, "Shared").def_readonly("value", &Shared::value); diff --git a/tests/test_interop.py b/tests/test_interop.py index 751a80e8e1..477880d413 100644 --- a/tests/test_interop.py +++ b/tests/test_interop.py @@ -5,6 +5,7 @@ import gc import itertools import sys +import sysconfig import threading import time import weakref @@ -14,7 +15,12 @@ import test_interop_2 as t2 import test_interop_3 as t3 +import env + free_threaded = hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() +types_are_immortal = sys.implementation.name in ("graalpy", "pypy") or ( + sysconfig.get_config_var("Py_GIL_DISABLED") and sys.version_info < (3, 14) +) # t1, t2, t3 all define bindings for the same C++ type `Shared`, as well as # several functions to create and inspect instances of that type. Each module @@ -53,16 +59,20 @@ def delattr_and_ensure_destroyed(*specs): @pytest.fixture(autouse=True) def clean_after(): yield - t3.clear_interop_bindings() - - delattr_and_ensure_destroyed( - *[ - (mod, name) - for mod in (t1, t2, t3) - for name in ("Shared", "SharedEnum", "RawShared") - if hasattr(mod, name) - ] - ) + if sys.implementation.name in ("pypy", "graalpy"): + pytest.gc_collect() + if sys.implementation.name != "graalpy": + t3.clear_interop_bindings() + + if not types_are_immortal: + delattr_and_ensure_destroyed( + *[ + (mod, name) + for mod in (t1, t2, t3) + for name in ("Shared", "SharedEnum", "RawShared") + if hasattr(mod, name) + ] + ) t1.pull_stats() t2.pull_stats() @@ -70,11 +80,19 @@ def clean_after(): def check_stats(mod, **entries): - if mod is None: + if mod is None or sys.implementation.name == "graalpy": + # graalpy seems to not do a full collection when we gc_collect(); there + # will be too few destructions in one check_stats() and then correspondingly + # too many in the next one for the same module, so just skip the check return if sys.implementation.name == "pypy": - gc.collect() + pytest.gc_collect() stats = mod.pull_stats() + if stats["move"] == entries.get("move", 0) + 1: + # Allow an extra move+destroy pair to account for older compilers + # not doing RVO like we expect. + entries["move"] = entries.get("move", 0) + 1 + entries["destroy"] = entries.get("destroy", 0) + 1 for name, value in entries.items(): assert stats.pop(name) == value assert all(val == 0 for val in stats.values()) @@ -176,11 +194,34 @@ def expect(from_mod, to_mod, pattern, **extra): check_stats(mod, **stats) -def test02_interop_unimported(): - # Before any types are bound, no to-Python conversions are possible - for mod in (t1, t2, t3): - expect(mod, mod, "none") +def test02a_interop_return_foreign_smart_holder(): + # Test a pybind11 domain returning a different pybind11 domain's type + # because it didn't have its own. + t2.bind_types() + t2.export_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t1.import_for_interop(t2.Shared) + t1.import_for_interop(t2.SharedEnum) + expect(t2, t1, "foreign") + expect(t1, t2, "local") # t2 is both source and dest + assert type(t1.make(1)) is t2.Shared + assert type(t2.make(2)) is t2.Shared + +def test02b_interop_return_foreign_shared_ptr(): + # Same thing but different holder type (t2 uses smart holder, t1 regular). + t1.bind_types() + t1.export_for_interop(t1.Shared) + t1.export_for_interop(t1.SharedEnum) + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t1.SharedEnum) + expect(t1, t3, "foreign") + expect(t3, t1, "local") # t1 is both source and dest + assert type(t1.make(1)) is t1.Shared + assert type(t3.make(2)) is t1.Shared + + +def test03_interop_unimported(): # Bind the types but don't share them yet t1.bind_types() t2.bind_types() @@ -205,7 +246,7 @@ def test02_interop_unimported(): expect(t2, t3, "isolated") -def test03_interop_import_export_errors(): +def test04_interop_import_export_errors(): t1.bind_types() t2.bind_types() t3.create_raw_binding() @@ -241,7 +282,11 @@ def test03_interop_import_export_errors(): t2.import_for_interop_wrong_type(t3.RawShared) -def test04_interop_exceptions(): +@pytest.mark.skipif( + (env.MACOS and env.PYPY) or env.ANDROID, + reason="same issue as test_exceptions.py test_cross_module_exception_translator", +) +def test05_interop_exceptions(): # Once t1 and t2 have registered with pymetabind, which happens as soon as # they each import or export anything, t1 can translate t2's exceptions. t1.bind_types() @@ -252,7 +297,7 @@ def test04_interop_exceptions(): t1.throw_shared(123) -def test05_interop_with_cpp(): +def test06_interop_with_cpp(): t1.bind_types() t2.bind_types() t3.create_raw_binding() @@ -300,36 +345,6 @@ def test05_interop_with_cpp(): assert type(t2.make(2)) is t2.Shared assert type(t3.make(3)) is t3.RawShared - # If we create a pybind11 Shared in t3, that takes priority over the raw one - t3.bind_types() - assert type(t3.make(4)) is t3.Shared - - -def test06_interop_return_foreign_smart_holder(): - # Test a pybind11 domain returning a different pybind11 domain's type - # because it didn't have its own. - t2.bind_types() - t2.export_for_interop(t2.Shared) - t2.export_for_interop(t2.SharedEnum) - t1.import_for_interop(t2.Shared) - t1.import_for_interop(t2.SharedEnum) - expect(t2, t1, "foreign") - expect(t1, t2, "local") # t2 is both source and dest - assert type(t1.make(1)) is t2.Shared - assert type(t2.make(2)) is t2.Shared - - -def test06_interop_return_foreign_shared_ptr(): - t1.bind_types() - t1.export_for_interop(t1.Shared) - t1.export_for_interop(t1.SharedEnum) - t2.import_for_interop(t1.Shared) - t2.import_for_interop(t1.SharedEnum) - expect(t1, t2, "foreign") - expect(t2, t1, "local") # t1 is both source and dest - assert type(t1.make(1)) is t1.Shared - assert type(t2.make(2)) is t1.Shared - def test07_interop_with_c(): t1.bind_types() @@ -342,6 +357,7 @@ def test07_interop_with_c(): expect(t3, t1, "foreign") +@pytest.mark.skipif(types_are_immortal, reason="can't GC type object on this platform") def test08_remove_binding(): t3.create_raw_binding() t2.import_for_interop_explicit(t3.RawShared) @@ -379,7 +395,7 @@ def test08_remove_binding(): # the type object. del t2.Shared.__pymetabind_binding__ del t2.SharedEnum.__pymetabind_binding__ - gc.collect() + pytest.gc_collect() expect(t2, t3, "isolated") expect(t3, t2, "isolated", enum=None) @@ -400,7 +416,7 @@ def test08_remove_binding(): # Removing the binding capsule should work just as well as removing # the type object. del t3.RawShared.__pymetabind_binding__ - gc.collect() + pytest.gc_collect() t3.export_raw_binding() # t3.RawShared was removed from the beginning of t3's list for Shared @@ -500,11 +516,35 @@ def repeatedly_attempt_conversions(): assert num_failed > 0 -def test11_implicit(): - # Create four different types of pyobject, all of which have C++ type Shared +def test11_import_export_all(): + # Enable automatic import and export in the t1/t2 domains. + # Still doesn't help with t3->t1/t2 since t3.RawShared is not a C++ type. + t1.import_all() + t1.export_all() t1.bind_types() + + t2.import_all() t2.bind_types() + t2.export_all() + + t3.create_raw_binding() + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t2.SharedEnum) + + expect(t1, t2, "foreign") + expect(t1, t3, "foreign", enum=False) # t3 didn't import all or import t1's enum + expect(t2, t1, "foreign") + expect(t2, t3, "isolated", enum=True) # t3 didn't import t2.Shared + t3.import_all() + expect(t2, t3, "foreign") # t3 didn't import t2.Shared + expect(t3, t1, "isolated", enum=True) + + +def test12_implicit(): + # Create four different types of pyobject, all of which have C++ type Shared t3.create_raw_binding() + t1.bind_types() + t2.bind_types() s1 = t1.make(10) s2 = t2.make(11) s3r = t3.make(12) @@ -537,25 +577,12 @@ def test11_implicit(): assert t2.check(s1) == 10 -def test12_import_export_all(): - # Enable automatic import and export in the t1/t2 domains. - # Still doesn't help with t3->t1/t2 since t3.RawShared is not a C++ type. - t1.import_all() - t1.export_all() - t1.bind_types() - - t2.import_all() - t2.bind_types() - t2.export_all() - - t3.create_raw_binding() - t3.import_for_interop(t1.Shared) - t3.import_for_interop(t2.SharedEnum) - - expect(t1, t2, "foreign") - expect(t1, t3, "foreign", enum=False) # t3 didn't import all or import t1's enum - expect(t2, t1, "foreign") - expect(t2, t3, "isolated", enum=True) # t3 didn't import t2.Shared - t3.import_all() - expect(t2, t3, "foreign") # t3 didn't import t2.Shared - expect(t3, t1, "isolated", enum=True) +if sys.implementation.name == "graalpy": + # gc.collect() on GraalPy doesn't reliably collect all garbage, even if + # it's run multiple times, so we can't remove bindings once we've created + # them. Reduce the test suite to a set that can cope with that. + del test03_interop_unimported + del test06_interop_with_cpp + del test07_interop_with_c + del test11_import_export_all + del test12_implicit diff --git a/tests/test_interop_1.cpp b/tests/test_interop_1.cpp index 5e4cd0071a..d3f6693e3c 100644 --- a/tests/test_interop_1.cpp +++ b/tests/test_interop_1.cpp @@ -18,8 +18,9 @@ namespace py = pybind11; PYBIND11_MODULE(test_interop_1, m, py::mod_gil_not_used()) { + py::handle hm = m; Shared::bind_funcs(m); - m.def("bind_types", [hm = py::handle(m)]() { Shared::bind_types(hm); }); + m.def("bind_types", [hm]() { Shared::bind_types(hm); }); m.def("throw_shared", [](int v) { throw SharedExc{v}; }); diff --git a/tests/test_interop_2.cpp b/tests/test_interop_2.cpp index f31852748a..1a4987fd82 100644 --- a/tests/test_interop_2.cpp +++ b/tests/test_interop_2.cpp @@ -18,8 +18,9 @@ namespace py = pybind11; PYBIND11_MODULE(test_interop_2, m, py::mod_gil_not_used()) { + py::handle hm = m; Shared::bind_funcs(m); - m.def("bind_types", [hm = py::handle(m)]() { Shared::bind_types(hm); }); + m.def("bind_types", [hm]() { Shared::bind_types(hm); }); py::register_exception_translator([](std::exception_ptr p) { try { std::rethrow_exception(p); diff --git a/tests/test_interop_3.cpp b/tests/test_interop_3.cpp index 4c25631c17..6ab6be0266 100644 --- a/tests/test_interop_3.cpp +++ b/tests/test_interop_3.cpp @@ -11,11 +11,12 @@ // modules from each other and from the rest of the pybind11 tests #define PYBIND11_INTERNALS_VERSION 300 -#include #include #include "test_interop.h" +#include + namespace py = pybind11; // The following is a manual binding to `struct Shared`, created using the @@ -32,14 +33,18 @@ struct raw_shared_instance { }; static void Shared_dealloc(struct raw_shared_instance *self) { - if (self->spacer[0] != 0x5a5a5a5a || self->spacer[1] != 0xa5a5a5a5) + if (self->spacer[0] != 0x5a5a5a5a || self->spacer[1] != 0xa5a5a5a5) { std::terminate(); // instance corrupted - if (self->weakrefs) + } + if (self->weakrefs) { PyObject_ClearWeakRefs((PyObject *) self); - if (self->destroy) + } + if (self->destroy) { self->ptr->~Shared(); - if (self->deallocate) + } + if (self->deallocate) { free(self->ptr); + } PyTypeObject *tp = Py_TYPE((PyObject *) self); PyObject_Free(self); @@ -47,8 +52,7 @@ static void Shared_dealloc(struct raw_shared_instance *self) { } static PyObject *Shared_new(PyTypeObject *type, Shared *value, pymb_rv_policy rvp) { - struct raw_shared_instance *self; - self = PyObject_New(raw_shared_instance, type); + struct raw_shared_instance *self = PyObject_New(raw_shared_instance, type); if (self) { memset((char *) self + sizeof(PyObject), 0, sizeof(*self) - sizeof(PyObject)); self->spacer[0] = 0x5a5a5a5a; @@ -95,8 +99,9 @@ static void *hook_from_python(pymb_binding *binding, uint8_t, void (*)(void *ctx, PyObject *obj), void *) noexcept { - if (binding->pytype != Py_TYPE(pyobj)) + if (binding->pytype != Py_TYPE(pyobj)) { return nullptr; + } return ((raw_shared_instance *) pyobj)->ptr; } @@ -105,26 +110,29 @@ static PyObject *hook_to_python(pymb_binding *binding, enum pymb_rv_policy rvp, pymb_to_python_feedback *feedback) noexcept { feedback->relocate = 0; - if (rvp == pymb_rv_policy_none) + if (rvp == pymb_rv_policy_none) { return nullptr; + } feedback->is_new = 1; return Shared_new(binding->pytype, (Shared *) cobj, rvp); } -static void hook_ignore_foreign_binding(pymb_binding *) noexcept {} -static void hook_ignore_foreign_framework(pymb_framework *) noexcept {} +static int hook_no_keep_alive(PyObject *, void *, void (*)(void *)) noexcept { return 0; } +static void hook_ignore_binding(pymb_binding *) noexcept {} +static void hook_ignore_framework(pymb_framework *) noexcept {} +static void hook_free_local_binding(pymb_binding *binding) noexcept { delete binding; } PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { static PyMemberDef Shared_members[] = { {"__weaklistoffset__", - Py_T_PYSSIZET, + T_PYSSIZET, offsetof(struct raw_shared_instance, weakrefs), - Py_READONLY, + READONLY, nullptr}, {nullptr, 0, 0, 0, nullptr}, }; static PyType_Slot Shared_slots[] = { - {Py_tp_doc, (void *) "Shared object"}, + {Py_tp_doc, (void *) const_cast("Shared object")}, {Py_tp_init, (void *) Shared_init}, {Py_tp_dealloc, (void *) Shared_dealloc}, {Py_tp_members, (void *) Shared_members}, @@ -139,8 +147,9 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { }; static auto *registry = pymb_get_registry(); - if (!registry) + if (!registry) { throw py::error_already_set(); + } static auto *fw = new pymb_framework{}; fw->name = "example framework for pybind11 tests"; @@ -148,26 +157,28 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { fw->abi_lang = pymb_abi_lang_c; fw->from_python = hook_from_python; fw->to_python = hook_to_python; - fw->keep_alive = [](PyObject *, void *, void (*)(void *)) noexcept { return 0; }; - fw->remove_local_binding = [](pymb_binding *) noexcept {}; - fw->free_local_binding = [](pymb_binding *binding) noexcept { delete binding; }; - fw->add_foreign_binding = hook_ignore_foreign_binding; - fw->remove_foreign_binding = hook_ignore_foreign_binding; - fw->add_foreign_framework = hook_ignore_foreign_framework; - fw->remove_foreign_framework = hook_ignore_foreign_framework; + fw->keep_alive = hook_no_keep_alive; + fw->remove_local_binding = hook_ignore_binding; + fw->free_local_binding = hook_free_local_binding; + fw->add_foreign_binding = hook_ignore_binding; + fw->remove_foreign_binding = hook_ignore_binding; + fw->add_foreign_framework = hook_ignore_framework; + fw->remove_foreign_framework = hook_ignore_framework; pymb_add_framework(registry, fw); int res = Py_AtExit(+[]() { pymb_remove_framework(fw); delete fw; }); - if (res != 0) + if (res != 0) { throw py::error_already_set(); + } + py::handle hm = m; Shared::bind_funcs(m); - m.def("bind_types", [hm = py::handle(m)]() { Shared::bind_types(hm); }); + m.def("bind_types", [hm]() { Shared::bind_types(hm); }); - m.def("export_raw_binding", [hm = py::handle(m)]() { + m.def("export_raw_binding", [hm]() { auto type = hm.attr("RawShared"); auto *binding = new pymb_binding{}; binding->framework = fw; @@ -177,10 +188,15 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { py::import_for_interop(type); }); - m.def("create_raw_binding", [hm = py::handle(m)]() { + m.def("create_raw_binding", [hm]() { + if (py::hasattr(hm, "RawShared")) { + hm.attr("export_raw_binding")(); + return; + } auto *type = (PyTypeObject *) PyType_FromSpec(&Shared_spec); - if (!type) + if (!type) { throw py::error_already_set(); + } #if PY_VERSION_HEX < 0x03090000 // __weaklistoffset__ member wasn't parsed until 3.9 type->tp_weaklistoffset = offsetof(struct raw_shared_instance, weakrefs); @@ -189,11 +205,12 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { hm.attr("export_raw_binding")(); }); - m.def("clear_interop_bindings", [hm = py::handle(m)]() { + m.def("clear_interop_bindings", [hm]() { // NB: this is not a general purpose solution; the bindings removed // here won't be re-added if `import_all` is called py::list bound; pymb_lock_registry(registry); + // NOLINTNEXTLINE(modernize-use-auto) PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { bound.append(py::reinterpret_borrow((PyObject *) binding->pytype)); } @@ -216,6 +233,19 @@ PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { throw std::runtime_error("Could not remove bindings"); } + // Force trying to re-import/export everything the next time + // import/export all are called, in case we removed bindings + // in between unit tests but couldn't destroy their types + // (because types are immortal in this environment) + for (auto key : py::detail::get_python_state_dict()) { + if (key.attr("startswith")("__pybind11").cast() + && key.attr("endswith")("interop").cast()) { + py::capsule cap = py::detail::get_python_state_dict()[key]; + py::detail::interop_internals **ii = cap; + (*ii)->import_all = (*ii)->export_all = false; + } + } + // Restore the ability for our own create_shared() etc to work // properly, since that's a foreign type relationship too if (py::hasattr(hm, "RawShared")) {