diff --git a/.clang-tidy b/.clang-tidy index 3a1995c326..200c3e3660 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -78,3 +78,4 @@ CheckOptions: value: true HeaderFilterRegex: 'pybind11/.*h' +ExcludeHeaderFilterRegex: 'pybind11/contrib/.*h' diff --git a/.codespell-ignore-lines b/.codespell-ignore-lines index e8cbf31447..c03cd2f727 100644 --- a/.codespell-ignore-lines +++ b/.codespell-ignore-lines @@ -2,6 +2,7 @@ template auto &this_ = static_cast(*this); if (load_impl(temp, false)) { + return load_impl(src, false); ssize_t nd = 0; auto trivial = broadcast(buffers, nd, shape); auto ndim = (size_t) nd; diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8435fbac94..eaedbf5b20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ ci: autoupdate_schedule: monthly # third-party content -exclude: ^tools/JoinPaths.cmake$ +exclude: ^(tools/JoinPaths.cmake|include/pybind11/contrib/.*)$ repos: diff --git a/CMakeLists.txt b/CMakeLists.txt index ba5f665a2f..803f26bd45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,12 +180,14 @@ if(PYBIND11_MASTER_PROJECT) endif() set(PYBIND11_HEADERS + include/pybind11/contrib/pymetabind.h include/pybind11/detail/class.h include/pybind11/detail/common.h include/pybind11/detail/cpp_conduit.h include/pybind11/detail/descr.h include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h include/pybind11/detail/exception_translation.h + include/pybind11/detail/foreign.h include/pybind11/detail/function_record_pyobject.h include/pybind11/detail/init.h include/pybind11/detail/internals.h diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 14bfc0bcdb..faaba38b8d 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -972,9 +972,14 @@ Module-local class bindings =========================== When creating a binding for a class, pybind11 by default makes that binding -"global" across modules. What this means is that a type defined in one module -can be returned from any module resulting in the same Python type. For -example, this allows the following: +"global" across modules. What this means is that instances whose type is +defined with a ``py::class_`` statement in one module can be passed to or +returned from a function defined in any other module that is "ABI compatible" +with the first, i.e., that was built with sufficiently similar versions of +pybind11 and of the C++ compiler and C++ standard library. The internal data +structures that pybind11 uses to keep track of its types and instances are +shared just as they would be if everything were in the same module. +For example, this allows the following: .. code-block:: cpp diff --git a/docs/advanced/interop.rst b/docs/advanced/interop.rst new file mode 100644 index 0000000000..64e8c06799 --- /dev/null +++ b/docs/advanced/interop.rst @@ -0,0 +1,296 @@ +.. _interop: + +Interoperating with foreign bindings +==================================== + +When you bind a function with pybind11 that has a parameter of type ``T``, +its typical behavior (if ``T`` does not use a :ref:`built-in ` +or :ref:`custom type caster `) is to only accept arguments +for that parameter that are Python instances of the type created by a +``py::class_(...)`` binding statement, or that derive from that type, +or that match a defined :ref:`implicit conversion ` +to that type (``py::implicitly_convertible()``). Moreover, +if the ``py::class_(...)`` binding statement was written in a different +pybind11 extension than the function that needs the ``T``, the two extensions +must be ABI-compatible: they must use similar enough versions of pybind11 that +it's safe for their respective copies of pybind11 to share their data +structures with each other. + +Sometimes, you might want more flexibility than that: + +- Perhaps you have a large codebase containing a number of different pybind11 + extension modules that share types with each other, and you want to upgrade + to a new and ABI-incompatible release of pybind11 in some fashion other than + "upgrade every module at the same time". + +- Perhaps you need to work with types provided by a third-party extension + such as PyTorch, which uses pybind11 but not the version you prefer. + +- Perhaps you'd like to port some of the especially performance-sensitive + parts of your bindings to a faster but less featureful binding framework, + without leaving the comfortable world of pybind11 behind entirely. + +To handle such situations, pybind11 can be taught to interoperate with bindings +that were not created using pybind11, or that were created with an +ABI-incompatible version of pybind11 (as long as it is new enough to support +this feature). For example, you can define a class binding for ``Pet`` in one +extension that is written using pybind11, and then write function bindings for +``void groom(Pet&)`` and ``Pet clone(const Pet&)`` in a separate extension +module that is written using `nanobind `__, or +vice versa. The interoperability mechanism described here allows each framework +to figure out (among other things) how to get a reference to a C++ ``Pet`` out +of a Python object provided by the other framework that supposedly contains a +Pet, without knowing anything about how that framework lays out its instances. +From pybind11's perspective, nanobind and its bindings are considered "foreign". + +In order for pybind11 to interoperate with another framework in this way, the +other framework must support the `pymetabind +`__ standard. See that link for +a list of frameworks that claim to do so. + +Exporting pybind11 bindings for other frameworks to use +------------------------------------------------------- + +In order for a type bound by pybind11 to be usable by other binding frameworks, +pybind11 must allocate a small data structure describing how others should work +with that type. While the overhead of this is low, it is not zero, so pybind11 +only does so for types where you request it. Pass the Python type object to +``py::export_for_interop()``, or use ``py::interoperate_by_default()`` if you +want all types to be exported automatically as soon as they are bound. + +You can use ``py::type::of()`` to get the Python type object for +a C++ type. For example: + +.. code-block:: cpp + + PYBIND11_MODULE(my_ext, m) { + auto pet = py::class_(m, "Pet") + .def(py::init()) + .def("speak", &Pet::speak); + + // These two lines are equivalent: + py::export_for_interop(pet); + py::export_for_interop(py::type::of()); + } + + +Importing other frameworks' bindings for pybind11 to use +-------------------------------------------------------- + +In order for pybind11 to interoperate with a foreign type, the foreign framework +that bound the type must have created an interoperability record for it. +Depending on the framework, this might occur automatically or might require +an operation similar to the ``py::export_for_interop()`` described in the +previous section. (You can tell if this has happened by checking for the +presence of an attribute on the type object called ``__pymetabind_binding__``.) +Consult the other framework's documentation for details. + +Once that's done, you can teach pybind11 about the foreign type by passing its +Python type object to ``py::import_for_interop()``. +This function takes an optional template argument specifying which C++ type to +associate the Python type with. If the foreign type was bound using another +C++ framework, such as nanobind or a different version of pybind11, the template +argument need not be provided because the C++ ``std::type_info`` structure +describing the type can be found by looking at the interoperability record. +On the other hand, if the foreign type is not written in C++ or is bound by +a non-C++ framework that doesn't know about ``std::type_info``, pybind11 won't +be able to figure out what the C++ type is, and needs you to specify it via +a template argument to ``py::import_for_inteorp()``. + +If you *don't* supply a template argument (for importing a C++ type), then +pybind11 will check for you that the binding you're adding was compiled using a +platform C++ ABI that is consistent with the build options for your pybind11 +extension. This helps to ensure that the exporter and importer mean the same +thing when they say, for example, ``std::vector``. +The import will throw an exception if an incompatibility is detected. + +If you *do* supply a template argument (for importing a +different-language type and specifying the C++ equivalent), pybind11 +will assume that you have validated compatibility yourself. Getting it +wrong can cause crashes and other sorts of undefined behavior, so if +you're working with bindings that were created in another language, make +doubly sure you're specifying a C++ type that is fully ABI-compatible with +the one used by the foreign binding. + +You can use ``py::interoperate_by_default()`` if you want pybind11 to +automatically import every compatible C++ type as soon as it has been +exported by another framework. + +.. code-block:: cpp + + // --- pet.h --- + #pragma once + #include + + struct Pet { + std::string name; + std::string sound; + + Pet(std::string _name, std::string _sound) + : name(std::move(_name)), sound(std::move(_sound)) {} + + std::string speak() const { return name + " goes " + sound + "!"; } + }; + + // --- pets.cc --- + #include + #include + #include "pet.h" + + NB_MODULE(pets, m) { + auto pet = nanobind::class_(m, "Pet") + .def(nanobind::init()) + .def("speak", &Pet::speak); + + nanobind::export_for_interop(pet); + } + + // --- groomer.cc --- + #include + #include "pet.h" + + std::string groom(const Pet& pet) { + return pet.name + " got a haircut"; + } + + PYBIND11_MODULE(groomer, m) { + auto pet = pybind11::module_::import_("pets").attr("Pet"); + + // This could go either before or after the function definition that + // relies on it + pybind11::import_for_interop(pet); + + // If Pet were bound by a non-C++ framework, you would instead say: + // pybind11::import_for_interop(pet); + + m.def("groom", &groom); + } + + +Automatic communication +----------------------- + +In large binding projects, you might prefer to share *all* types rather than +only those you nominate. For that, pybind11 provides the +``py::interoperate_by_default()`` function. It takes two optional bool +parameters that specify whether you want automatic export and/or automatic +import; if you don't specify the parameters, then both are enabled. + +Automatic export is equivalent to writing a call to ``py::export_for_interop()`` +after every ``py::class_``, ``py::enum_``, or ``py::native_enum`` binding +statement in any pybind11 module that is ABI-compatible with the one in which +you wrote the call. + +Automatic import is equivalent to writing a call to ``py::import_for_interop()`` +after every export of a type from a different framework. It only import +bindings written in C++ with a compatible platform ABI (the same ones that +``py::import_for_interop()`` can import without a template argument); +bindings written in other languages must always be imported explicitly. + +Automatic import and export apply both to types that already exist and +types that will be bound in the future. They cannot be disabled once enabled. + +Here is the above example recast to use automatic communication. + +.. code-block:: cpp + + // (pet.h unchanged) + + // --- pets.cc --- + #include + #include + #include "pet.h" + + NB_MODULE(pets, m) { + nanobind::interoperate_by_default(); + nanobind::class_(m, "Pet") + .def(nanobind::init()) + .def("speak", &Pet::speak); + } + + // --- groomer.cc --- + #include + #include "pet.h" + + std::string groom(const Pet& pet) { + return pet.name + " got a haircut"; + } + + PYBIND11_MODULE(groomer, m) { + pybind11::interoperate_by_default(); + m.def("groom", &groom); + } + + +Conversion semantics and caveats +-------------------------------- + +Cross-framework inheritance is not supported: a type bound +using pybind11 must only have base classes that were bound using +ABI-compatible versions of pybind11. + +A function bound using pybind11 cannot perform a conversion to +``std::unique_ptr`` using a foreign binding for ``T``, because the +interoperability mechanism doesn't provide any way to ask a foreign instance +to relinquish its ownership. + +When converting from a foreign instance to ``std::shared_ptr``, pybind11 +generally cannot "see inside" the instance to find an existing ``shared_ptr`` +to share ownership with, so it will create a new ``shared_ptr`` control block +that owns a reference to the Python object. This is usually not a problem, but +does mean that ``shared_ptr::use_count()`` won't work like you expect. (If +``T`` inherits ``std::enable_shared_from_this``, then pybind11 can use that +to find the existing ``shared_ptr``, and will do so instead.) + +Type casters (both :ref:`built-in ` and :ref:`custom +`) execute before the interoperability mechanism +has a chance to step in. pybind11 is not able to execute type casters from +a different framework; you will need to port them to a pybind11 equivalent. +Interoperability only helps with bindings, as produced by ``py::class_`` and +similar statements. + +:ref:`Implicit conversion ` defined using +``py::implicitly_convertible()`` can convert *from* foreign types. +Implicit conversions *to* a foreign type should be registered with its +binding library, not with pybind11. + +When a C++-to-foreign-Python conversion is performed in a context that does +not specify the ``return_value_policy``, the policy to use is inferred using +pybind11's rules, which may differ from the foreign framework's. + +It is possible for multiple foreign bindings to exist for the same C++ type, +or for a particular C++ type to have both a native pybind11 binding +and one or more foreign ones. This might occur due to separate Python +extensions each having their own need to bind a common type, as discussed in +the section on :ref:`module-local bindings `. In such cases, +pybind11 always tries bindings for a given C++ type ``T`` in the following order: + +* the pybind11 binding for ``T`` that was declared with ``py::module_local()`` + in this extension module, if any; then + +* the pybind11 binding for ``T`` that was declared without ``py::module_local()`` + in either this extension module or another ABI-compatible one (drawing no + distinction between the two), if any; then + +* if performing a from-Python conversion on an instance of a pybind11 binding + for ``T`` that was declared with ``py::module_local()`` in a different + but ABI-compatible module, that binding; otherwise + +* each known foreign binding, in the order in which they were imported, + without making any distinction between other versions of pybind11 and + non-pybind11 frameworks. (If automatic import is enabled, then the import + order will match the original export order.) + +You can use the interoperability mechanism to share :ref:`module-local bindings +` with other modules. Unlike the sharing that happens by default, +this allows you to return instances of such bindings from outside the module in +which they were defined. + +When performing C++-to-Python conversion of a type for which +:ref:`automatic downcasting ` is applicable, +the downcast occurs in the binding library that is originally performing the +conversion, even if the result will then be obtained using a foreign binding. +That means foreign frameworks returning pybind11 types might not downcast +them in the same way that pybind11 does; they might only be able to downcast +from a primary base (with no this-pointer adjustment / no multiple inheritance), +or not downcast at all. diff --git a/docs/index.rst b/docs/index.rst index 77b097c574..42ef24aadb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,6 +34,7 @@ advanced/smart_ptrs advanced/cast/index advanced/pycpp/index + advanced/interop advanced/embedding advanced/misc advanced/deprecated diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 9b373fc268..71b1f83d47 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,55 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +v3.1 +==== + +The major new feature in pybind11 v3.1 is support for +:ref:`interoperability with other binding frameworks ` and other +(future) versions of pybind11. See the linked documentation for details. + +This support was added in an ABI-compatible way, so you can combine pybind11 +v3.1 extensions with v3.0 extensions. Classes and enums bound using pybind11 +v3.1 support all interoperability features. Classes and ``py::enum_``\s bound +using pybind11 v3.0 can still be exported manually by a pybind11 v3.1 extension +calling ``py::export_for_interop()``, but they won't be exported automatically +and they can't be returned by value from a foreign binding. +``py::native_enum``\s bound using pybind11 v3.0 don't support the +interoperability mechanism at all. + +There is one implication of the new interoperability support that might result +in new compiler errors for some previously-working binding code. Previously, +pybind11 only attempted to call a bound type's copy constructor or move +constructor if that type was ever returned from a pybind11-bound function. +Now, pybind11 must allow for the possibility that a pybind11-bound type is +returned from a foreign framework's bound functions, so it will generate +code that's capable of calling copy and move constructors for any bound type +that satisfies ``std::is_{copy,move}_constructible``. There exist types that +satisfy that type trait but will produce errors if you actually try to copy +them, such as the following: + +.. code-block:: cpp + + struct MoveOnly { MoveOnly(MoveOnly&&) noexcept = default; } + struct Container { + std::vector items; + }; + +``Container`` in this example satisfies ``std::is_copy_constructible``, but +actually trying to copy it will fail at compile time because the vector element +``MoveOnly`` is not copyable. The solution is to explicitly mark ``Container`` +as move-only: + +.. code-block:: cpp + + struct MoveOnly { MoveOnly(MoveOnly&&) noexcept = default; } + struct Container { + Container() = default; + Container(Container&&) noexcept = default; + + std::vector items; + }; + .. _upgrade-guide-3.0: v3.0 diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index c635791fee..259866fc1c 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -84,45 +84,42 @@ class type_caster_enum_type { bool load(handle src, bool convert) { handle native_enum = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType))); - if (native_enum) { - if (!isinstance(src, native_enum)) { - return false; - } + if (native_enum && isinstance(src, native_enum)) { type_caster underlying_caster; if (!underlying_caster.load(src.attr("value"), convert)) { pybind11_fail("native_enum internal consistency failure."); } - value = static_cast(static_cast(underlying_caster)); + native_value = static_cast(static_cast(underlying_caster)); + native_loaded = true; return true; } - if (!pybind11_enum_) { - pybind11_enum_.reset(new type_caster_base()); + + type_caster_base legacy_caster; + if (legacy_caster.load(src, convert)) { + legacy_ptr = static_cast(legacy_caster); + return true; } - return pybind11_enum_->load(src, convert); + return false; } template using cast_op_type = detail::cast_op_type; // NOLINTNEXTLINE(google-explicit-constructor) - operator EnumType *() { - if (!pybind11_enum_) { - return &value; - } - return pybind11_enum_->operator EnumType *(); - } + operator EnumType *() { return native_loaded ? &native_value : legacy_ptr; } // NOLINTNEXTLINE(google-explicit-constructor) operator EnumType &() { - if (!pybind11_enum_) { - return value; + if (!native_loaded && !legacy_ptr) { + throw reference_cast_error(); } - return pybind11_enum_->operator EnumType &(); + return native_loaded ? native_value : *legacy_ptr; } private: - std::unique_ptr> pybind11_enum_; - EnumType value; + EnumType native_value; // if loading a py::native_enum + bool native_loaded = false; + EnumType *legacy_ptr = nullptr; // if loading a py::enum_ or foreign }; template @@ -862,6 +859,71 @@ struct holder_helper { static auto get(const T &p) -> decltype(p.get()) { return p.get(); } }; +struct holder_caster_foreign_helpers { + struct py_deleter { + void operator()(const void *) const noexcept { + // Don't run the deleter if the interpreter has been shut down + if (Py_IsInitialized() == 0) { + return; + } + gil_scoped_acquire guard; + Py_DECREF(o); + } + + PyObject *o; + }; + + template + static auto try_shared_from_this(type *value, std::shared_ptr *holder_out) + -> decltype(value->shared_from_this(), bool()) { + // object derives from enable_shared_from_this; + // try to reuse an existing shared_ptr if one is known + if (auto existing = try_get_shared_from_this(value)) { + *holder_out = std::static_pointer_cast(existing); + return true; + } + return false; + } + + template + static bool try_shared_from_this(void *, std::shared_ptr *) { + return false; + } + + template + static bool set_foreign_holder(handle src, type *value, std::shared_ptr *holder_out) { + // We only support using std::shared_ptr for foreign T, and + // it's done by creating a new shared_ptr control block that + // owns a reference to the original Python object. + if (value == nullptr) { + *holder_out = {}; + return true; + } + if (try_shared_from_this(value, holder_out)) { + return true; + } + *holder_out = std::shared_ptr(value, py_deleter{src.inc_ref().ptr()}); + return true; + } + + template + static bool + set_foreign_holder(handle src, const type *value, std::shared_ptr *holder_out) { + std::shared_ptr holder_mut; + if (set_foreign_holder(src, const_cast(value), &holder_mut)) { + *holder_out = holder_mut; + return true; + } + return false; + } + + template + static bool set_foreign_holder(handle, type *, ...) { + throw cast_error("Unable to cast foreign type to held instance -- " + "only std::shared_ptr is supported in this case"); + } +}; + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization. /// Type caster for holder types like std::shared_ptr, etc. /// The SFINAE hook is provided to help work around the current lack of support @@ -906,6 +968,10 @@ struct copyable_holder_caster : public type_caster_base { } } + bool set_foreign_holder(handle src) { + return holder_caster_foreign_helpers::set_foreign_holder(src, (type *) value, &holder); + } + void load_value(value_and_holder &&v_h) { if (v_h.holder_constructed()) { value = v_h.value_ptr(); @@ -976,7 +1042,7 @@ struct copyable_holder_caster< } explicit operator std::shared_ptr *() { - if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + if (sh_load_helper.was_populated) { pybind11_fail("Passing `std::shared_ptr *` from Python to C++ is not supported " "(inherently unsafe)."); } @@ -984,14 +1050,14 @@ struct copyable_holder_caster< } explicit operator std::shared_ptr &() { - if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + if (sh_load_helper.was_populated) { shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, value); } return shared_ptr_storage; } std::weak_ptr potentially_slicing_weak_ptr() { - if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + if (sh_load_helper.was_populated) { // Reusing shared_ptr code to minimize code complexity. shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, @@ -1005,15 +1071,12 @@ struct copyable_holder_caster< static handle cast(const std::shared_ptr &src, return_value_policy policy, handle parent) { const auto *ptr = src.get(); - auto st = type_caster_base::src_and_type(ptr); - if (st.second == nullptr) { - return handle(); // no type info: error will be set already - } - if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + typename type_caster_base::cast_sources srcs{ptr}; + if (srcs.creates_smart_holder()) { return smart_holder_type_caster_support::smart_holder_from_shared_ptr( - src, policy, parent, st); + src, policy, parent, srcs.result); } - return type_caster_base::cast_holder(ptr, &src); + return type_caster_base::cast_holder(srcs, &src); } // This function will succeed even if the `responsible_parent` does not own the @@ -1040,6 +1103,11 @@ struct copyable_holder_caster< } } + bool set_foreign_holder(handle src) { + return holder_caster_foreign_helpers::set_foreign_holder( + src, (type *) value, &shared_ptr_storage); + } + void load_value(value_and_holder &&v_h) { if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = v_h; @@ -1077,6 +1145,7 @@ struct copyable_holder_caster< value = cast.second(sub_caster.value); if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + sh_load_helper.was_populated = true; } else { shared_ptr_storage = std::shared_ptr(sub_caster.shared_ptr_storage, (type *) value); @@ -1183,21 +1252,12 @@ struct move_only_holder_caster< static handle cast(std::unique_ptr &&src, return_value_policy policy, handle parent) { auto *ptr = src.get(); - auto st = type_caster_base::src_and_type(ptr); - if (st.second == nullptr) { - return handle(); // no type info: error will be set already - } - if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + typename type_caster_base::cast_sources srcs{ptr}; + if (srcs.creates_smart_holder()) { return smart_holder_type_caster_support::smart_holder_from_unique_ptr( - std::move(src), policy, parent, st); + std::move(src), policy, parent, srcs.result); } - return type_caster_generic::cast(st.first, - return_value_policy::take_ownership, - {}, - st.second, - nullptr, - nullptr, - std::addressof(src)); + return type_caster_base::cast_holder(srcs, &src); } static handle @@ -1223,6 +1283,12 @@ struct move_only_holder_caster< return false; } + bool set_foreign_holder(handle) { + throw cast_error("Foreign types cannot be converted to std::unique_ptr " + "because we don't know how to make them relinquish " + "ownership"); + } + void load_value(value_and_holder &&v_h) { if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = v_h; @@ -1281,6 +1347,7 @@ struct move_only_holder_caster< value = cast.second(sub_caster.value); if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + sh_load_helper.was_populated = true; } else { pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); @@ -2338,11 +2405,11 @@ object object_api::call(Args &&...args) const { PYBIND11_NAMESPACE_END(detail) template -handle type::handle_of() { +handle type::handle_of(bool foreign_ok) { static_assert(std::is_base_of>::value, - "py::type::of only supports the case where T is a registered C++ types."); + "py::type::of only supports the case where T is a registered C++ type."); - return detail::get_type_handle(typeid(T), true); + return detail::get_type_handle(typeid(T), true, foreign_ok); } #define PYBIND11_MAKE_OPAQUE(...) \ diff --git a/include/pybind11/contrib/pymetabind.h b/include/pybind11/contrib/pymetabind.h new file mode 100644 index 0000000000..53a9186e99 --- /dev/null +++ b/include/pybind11/contrib/pymetabind.h @@ -0,0 +1,1148 @@ +/* + * pymetabind.h: definitions for interoperability between different + * Python binding frameworks + * + * Copy this header file into the implementation of a framework that uses it. + * This functionality is intended to be used by the framework itself, + * rather than by users of the framework. + * + * This is version 0.3+dev of pymetabind. Changelog: + * + * Unreleased: Properly return NULL if registry capsule creation fails. + * Support GraalPy. + * + * Version 0.3: Don't do a Py_DECREF in `pymb_remove_framework` since the + * 2025-09-15 interpreter might already be finalized at that point. + * Revamp binding lifetime logic. Add `remove_local_binding` + * and `free_local_binding` callbacks. + * Add `pymb_framework::registry` and use it to simplify + * the signatures of `pymb_remove_framework`, + * `pymb_add_binding`, and `pymb_remove_binding`. + * Update `to_python` protocol to be friendlier to + * pybind11 instances with shared/smart holders. + * Remove `pymb_rv_policy_reference_internal`; add + * `pymb_rv_policy_share_ownership`. Change `keep_alive` + * return value convention. + * + * Version 0.2: Use a bitmask for `pymb_framework::flags` and add leak_safe + * 2025-09-11 flag. Change `translate_exception` to be non-throwing. + * Add casts from PyTypeObject* to PyObject* where needed. + * Fix typo in Py_GIL_DISABLED. Add noexcept to callback types. + * Rename `hook` -> `link` in linked list nodes. + * Use `static inline` linkage in C. Free registry on exit. + * Clear list hooks when adding frameworks/bindings in case + * the user didn't zero-initialize. Avoid abi_extra string + * comparisons if the strings are already pointer-equal. + * Add `remove_foreign_framework` callback. + * + * Version 0.1: Initial draft. ABI may change without warning while we + * 2025-08-16 prove out the concept. Please wait for a 1.0 release + * before including this header in a published release of + * any binding framework. + * + * Copyright (c) 2025 Hudson River Trading + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include +#include +#include + +#if !defined(PY_VERSION_HEX) +# error You must include Python.h before this header +#endif + +// `inline` in C implies a promise to provide an out-of-line definition +// elsewhere; in C++ it does not. +#ifdef __cplusplus +# define PYMB_INLINE inline +#else +# define PYMB_INLINE static inline +#endif + +/* + * There are two ways to use this header file. The default is header-only style, + * where all functions are defined as `inline` (C++) / `static inline` (C). + * If you want to emit functions as non-inline, perhaps so you can link against + * them from non-C/C++ code, then do the following: + * - In every compilation unit that includes this header, `#define PYMB_FUNC` + * first. (The `PYMB_FUNC` macro will be expanded in place of the "inline" + * keyword, so you can also use it to add any other declaration attributes + * required by your environment.) + * - In all those compilation units except one, also `#define PYMB_DECLS_ONLY` + * before including this header. The definitions will be emitted in the + * compilation unit that doesn't request `PYMB_DECLS_ONLY`. + */ +#if !defined(PYMB_FUNC) +#define PYMB_FUNC PYMB_INLINE +#endif + +#if defined(__cplusplus) +#define PYMB_NOEXCEPT noexcept +extern "C" { +#else +#define PYMB_NOEXCEPT +#endif + +/* + * Approach used to cast a previously unknown native instance into a Python + * object. This is similar to `pybind11::return_value_policy` or + * `nanobind::rv_policy`; some different options are provided than those, + * but same-named enumerators have the same semantics and values. + */ +enum pymb_rv_policy { + // Create a Python object that wraps a pointer to a heap-allocated + // native instance and will destroy and deallocate it (in whatever way + // is most natural for the target language) when the Python object is + // destroyed + pymb_rv_policy_take_ownership = 2, + + // Create a Python object that owns a new native instance created by + // copying the given one + pymb_rv_policy_copy = 3, + + // Create a Python object that owns a new native instance created by + // moving the given one + pymb_rv_policy_move = 4, + + // Create a Python object that wraps a pointer to a native instance + // but will not destroy or deallocate it + pymb_rv_policy_reference = 5, + + // Create a Python object that wraps a pointer to a native instance + // and will perform a custom action when the Python object is destroyed. + // The custom action is specified using the first call to keep_alive() + // after the object is created, and such a call must occur in order for + // the object to be considered fully initialized. + pymb_rv_policy_share_ownership = 6, + + // Don't create a new Python object; only try to look up an existing one + // from the same framework + pymb_rv_policy_none = 7 +}; + +/* + * The language to which a particular framework provides bindings. Each + * language has its own semantics for how to interpret + * `pymb_framework::abi_extra` and `pymb_binding::native_type`. + */ +enum pymb_abi_lang { + // C. `pymb_framework::abi_extra` and `pymb_binding::native_type` are NULL. + pymb_abi_lang_c = 1, + + // C++. `pymb_framework::abi_extra` is in the format used by + // nanobind since 2.6.1 (NB_PLATFORM_ABI_TAG in nanobind/src/nb_abi.h) + // and pybind11 since 2.11.2/2.12.1/2.13.6 (PYBIND11_PLATFORM_ABI_ID in + // pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h). + // `pymb_binding::native_type` is a cast `const std::type_info*` pointer. + pymb_abi_lang_cpp = 2, + + // extensions welcome! +}; + +/* + * Simple linked list implementation. `pymb_list_node` should be the first + * member of a structure so you can downcast it to the appropriate type. + */ +struct pymb_list_node { + struct pymb_list_node *next; + struct pymb_list_node *prev; +}; + +struct pymb_list { + struct pymb_list_node head; +}; + +PYMB_INLINE void pymb_list_init(struct pymb_list* list) { + list->head.prev = list->head.next = &list->head; +} + +PYMB_INLINE void pymb_list_unlink(struct pymb_list_node* node) { + if (node->next) { + node->next->prev = node->prev; + node->prev->next = node->next; + node->next = node->prev = NULL; + } +} + +PYMB_INLINE void pymb_list_append(struct pymb_list* list, + struct pymb_list_node* node) { + pymb_list_unlink(node); + struct pymb_list_node* tail = list->head.prev; + tail->next = node; + list->head.prev = node; + node->prev = tail; + node->next = &list->head; +} + +PYMB_INLINE int pymb_list_is_empty(struct pymb_list* list) { + return list->head.next == &list->head; +} + +#define PYMB_LIST_FOREACH(type, name, list) \ + for (type name = (type) (list).head.next; \ + name != (type) &(list).head; \ + name = (type) name->link.next) + +/* + * The registry holds information about all the interoperable binding + * frameworks and individual type bindings that are loaded in a Python + * interpreter process. It is protected by a mutex in free-threaded builds, + * and by the GIL in regular builds. + * + * The only data structure we use is a C doubly-linked list, which offers a + * lowest-common-denominator ABI and cheap addition and removal. It is expected + * that individual binding frameworks will use their `add_foreign_binding` and + * `remove_foreign_binding` callbacks to maintain references to these structures + * in more-performant private data structures of their choosing. + * + * The pointer to the registry is stored in a Python capsule object with type + * "pymetabind_registry", which is stored in the PyInterpreterState_GetDict() + * under the string key "__pymetabind_registry__". Any ABI-incompatible changes + * after v1.0 (which we hope to avoid!) will result in a new name for the + * dictionary key. You can obtain a registry pointer using + * `pymb_get_registry()`, defined below. + */ +struct pymb_registry { + // Linked list of registered `pymb_framework` structures + struct pymb_list frameworks; + + // Linked list of registered `pymb_binding` structures + struct pymb_list bindings; + + // Heap-allocated PyMethodDef for bound type weakref callback + PyMethodDef* weakref_callback_def; + + // Reserved for future extensions; currently set to 0 + uint16_t reserved; + + // Set to true when the capsule that points to this registry is destroyed + uint8_t deallocate_when_empty; + +#if defined(Py_GIL_DISABLED) + // Mutex guarding accesses to `frameworks` and `bindings`. + // On non-free-threading builds, these are guarded by the Python GIL. + PyMutex mutex; +#endif +}; + +#if defined(Py_GIL_DISABLED) +PYMB_INLINE void pymb_lock_registry(struct pymb_registry* registry) { + PyMutex_Lock(®istry->mutex); +} +PYMB_INLINE void pymb_unlock_registry(struct pymb_registry* registry) { + PyMutex_Unlock(®istry->mutex); +} +#else +PYMB_INLINE void pymb_lock_registry(struct pymb_registry* registry) { + (void) registry; +} +PYMB_INLINE void pymb_unlock_registry(struct pymb_registry* registry) { + (void) registry; +} +#endif + +struct pymb_binding; + +/* Flags for a `pymb_framework` */ +enum pymb_framework_flags { + // Does this framework guarantee that its `pymb_binding` structures remain + // valid to use for the lifetime of the Python interpreter process once + // they have been linked into the lists in `pymb_registry`? Setting this + // flag reduces the number of atomic operations needed to work with + // this framework's bindings in free-threaded builds. + pymb_framework_bindings_usable_forever = 0x0001, + + // Does this framework reliably deallocate all of its type and function + // objects by the time the Python interpreter is finalized, in the absence + // of bugs in user code? If not, it might cause leaks of other frameworks' + // types or functions, via attributes or default argument values for + // this framework's leaked objects. Other frameworks can suppress their + // leak warnings (if so equipped) when a non-`leak_safe` framework is added. + pymb_framework_leak_safe = 0x0002, +}; + +/* Additional results from `pymb_framework::to_python` */ +struct pymb_to_python_feedback { + // Ignored on entry. On exit, set to 1 if the returned Python object + // was created by the `to_python` call, or zero if it already existed and + // was simply looked up. + uint8_t is_new; + + // On entry, indicates whether the caller can control whether the native + // instance `cobj` passed to `to_python` is destroyed after the conversion: + // set to 1 if a relocation is allowable or 0 if `cobj` must be destroyed + // after the call. (This is only relevant when using pymb_rv_policy_move.) + // On exit, set to 1 if destruction should be inhibited because `*cobj` + // was relocated into the new instance. Must be left as zero on exit if + // set to zero on entry. + uint8_t relocate; +}; + +/* + * Information about one framework that has registered itself with pymetabind. + * "Framework" here refers to a set of bindings that are natively mutually + * interoperable. So, different binding libraries would be different frameworks, + * as would versions of the same library that use incompatible data structures + * due to ABI changes or build flags. + * + * A framework that wishes to either export bindings (allow other frameworks + * to perform to/from Python conversion for its types) or import bindings + * (perform its own to/from Python conversion for other frameworks' types) + * must start by creating and filling out a `pymb_framework` structure. + * This can be allocated in any way that the framework prefers (e.g., on + * the heap or in static storage). Once filled out, the framework structure + * should be passed to `pymb_add_framework()`. It must then remain accessible + * and unmodified (except as documented below) until the Python interpreter + * is finalized. After finalization, such as in a `Py_AtExit` handler, if + * all bindings have been removed already, you may optionally clean up by + * calling `pymb_remove_framework()` and then deallocating the + * `pymb_framework` structure. + * + * All fields of this structure are set before it is made visible to other + * threads and then never changed, so they don't need locking to access. + * Some methods require locking or other synchronization to call; see their + * individual documentation. + */ +struct pymb_framework { + // Links to the previous and next framework in the list of + // `pymb_registry::frameworks`. May be modified as other frameworks are + // added; protected by the `pymb_registry::mutex` in free-threaded builds. + struct pymb_list_node link; + + // Link to the `pymb_registry` that this framework is registered with. + // Filled in by `pymb_add_framework()`. + struct pymb_registry* registry; + + // Human-readable description of this framework, as a NUL-terminated string + const char* name; + + // Flags for this framework, a combination of `enum pymb_framework_flags`. + // Undefined flags must be set to zero to allow for future + // backward-compatible extensions. + uint16_t flags; + + // Reserved for future extensions. Set to 0. + uint8_t reserved[2]; + + // The language to which this framework provides bindings: one of the + // `pymb_abi_lang` enumerators. + enum pymb_abi_lang abi_lang; + + // NUL-terminated string constant encoding additional information that must + // match in order for two types with the same `abi_lang` to be usable from + // each other's environments. See documentation of `abi_lang` enumerators + // for language-specific guidance. This may be NULL if there are no + // additional ABI details that are relevant for your language. + // + // This is only the platform details that affect things like the layout + // of objects provided by the `abi_lang` (std::string, etc); Python build + // details (free-threaded, stable ABI, etc) should not impact this string. + // Details that are already guaranteed to match by virtue of being in the + // same address space -- architecture, pointer size, OS -- also should not + // impact this string. + // + // For efficiency, `pymb_add_framework()` will compare this against every + // other registered framework's `abi_extra` tag, and re-point an incoming + // framework's `abi_extra` field to refer to the matching `abi_extra` string + // of an already-registered framework if one exists. This acts as a simple + // form of interning to speed up checking that a given binding is usable. + // Thus, to check whether another framework's ABI matches yours, you can + // do a pointer comparison `me->abi_extra == them->abi_extra`. + const char* abi_extra; + + // The function pointers below allow other frameworks to interact with + // bindings provided by this framework. They are constant after construction + // and must not throw C++ exceptions. They must not be NULL; if a feature + // is not relevant to your use case, provide a stub that always fails. + + // Extract a C/C++/etc object from `pyobj`. The desired type is specified by + // providing a `pymb_binding*` for some binding that belongs to this + // framework. Return a pointer to the object, or NULL if no pointer of the + // appropriate type could be extracted. + // + // If `convert` is nonzero, be more willing to perform implicit conversions + // to make the cast succeed; the intent is that one could perform overload + // resolution by doing a first pass with convert=false to find an exact + // match, and then a second with convert=true to find an approximate match + // if there's no exact match. + // + // If `keep_referenced` is not NULL, then `from_python` may make calls to + // `keep_referenced` to request that some Python objects remain referenced + // until the returned object is no longer needed. The `keep_referenced_ctx` + // will be passed as the first argument to any such calls. + // `keep_referenced` should incref its `obj` immediately and remember + // that it should be decref'ed later, for no net change in refcount. + // This is an abstraction around something like the cleanup_list in + // nanobind or loader_life_support in pybind11. The pointer returned by + // `from_python` may be invalidated once the `keep_referenced` references + // are dropped. If you're converting a function argument, you should keep + // any `keep_referenced` references alive until the function returns. + // If you're converting for some other purpose, you probably want to make + // a copy of the object to which `from_python`'s return value points before + // you drop the references. + // + // On free-threaded builds, no direct synchronization is required to call + // this method, but you must ensure the `binding` won't be destroyed during + // (or before) your call. This generally requires maintaining a continuously + // attached Python thread state whenever you hold a pointer to `binding` + // that a concurrent call to your framework's `remove_foreign_binding` + // method wouldn't be able to clear. See the comment for `pymb_binding`. + void* (*from_python)(struct pymb_binding* binding, + PyObject* pyobj, + uint8_t convert, + void (*keep_referenced)(void* ctx, PyObject* obj), + void* keep_referenced_ctx) PYMB_NOEXCEPT; + + // Wrap the C/C++/etc object `cobj` into a Python object using the given + // return value policy. The type is specified by providing a `pymb_binding*` + // for some binding that belongs to this framework. + // + // The semantics of this function are as follows: + // - If there is already a live Python object created by this framework for + // this C++ object address and type, it will be returned and the `rvp` is + // ignored. + // - Otherwise, if `rvp == pymb_rv_policy_none`, NULL is returned without + // the Python error indicator set. + // - Otherwise, a new Python object will be created and returned. It will + // wrap either the pointer `cobj` or a copy/move of the contents of + // `cobj`, depending on the value of `rvp`. + // + // Returns a new reference to a Python object, or NULL if not possible. + // Also sets *feedback to provide additional information about the + // conversion. + // + // After a successful `to_python` call that returns a new instance and + // used `pymb_rv_policy_share_ownership`, the caller must make a call to + // `keep_alive` to describe how the shared ownership should be managed. + // + // On free-threaded builds, no direct synchronization is required to call + // this method, but you must ensure the `binding` won't be destroyed during + // (or before) your call. This generally requires maintaining a continuously + // attached Python thread state whenever you hold a pointer to `binding` + // that a concurrent call to your framework's `remove_foreign_binding` + // method wouldn't be able to clear. See the comment for `pymb_binding`. + PyObject* (*to_python)(struct pymb_binding* binding, + void* cobj, + enum pymb_rv_policy rvp, + struct pymb_to_python_feedback* feedback) PYMB_NOEXCEPT; + + // Request that a PyObject reference be dropped, or that a callback + // be invoked, when `nurse` is destroyed. `nurse` should be an object + // whose type is bound by this framework. If `cb` is NULL, then + // `payload` is a PyObject* to decref; otherwise `payload` will + // be passed as the argument to `cb`. Returns 1 if successful, + // 0 on error. This method may always return 0 if the framework has + // no better way to do a keep-alive than by creating a weakref; + // it is expected that the caller can handle creating the weakref. + // + // No synchronization is required to call this method. + int (*keep_alive)(PyObject* nurse, + void* payload, + void (*cb)(void*)) PYMB_NOEXCEPT; + + // Attempt to translate the native exception `eptr` into a Python exception. + // If `abi_lang` is C++, then `eptr` should be cast to `std::exception_ptr*` + // before use; semantics for other languages have not been defined yet. This + // should translate only framework-specific exceptions or user-defined + // exceptions that were registered with the framework, not generic ones + // such as `std::exception`. If translation succeeds, return 1 with the + // Python error indicator set; otherwise, return 0. An exception may be + // converted into a different exception by modifying `*eptr` and returning + // zero. This method may be set to NULL if its framework does not have + // a concept of exception translation. + // + // No synchronization is required to call this method. + int (*translate_exception)(void* eptr) PYMB_NOEXCEPT; + + // Notify this framework that one of its own bindings is being removed. + // This will occur synchronously from within a call to + // `pymb_remove_binding()`. Don't free the binding yet; wait for a later + // call to `free_local_binding`. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*remove_local_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; + + // Request this framework to free one of its own binding structures. + // A call to `pymb_remove_binding()` will eventually result in a call to + // this method, once pymetabind can prove no one is concurrently using the + // binding anymore. + // + // No synchronization is required to call this method. + void (*free_local_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; + + // Notify this framework that some other framework published a new binding. + // This call will be made after the new binding has been linked into the + // `pymb_registry::bindings` list. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*add_foreign_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; + + // Notify this framework that some other framework is about to remove + // a binding. This call will be made after the binding has been removed + // from the `pymb_registry::bindings` list. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*remove_foreign_binding)(struct pymb_binding* binding) PYMB_NOEXCEPT; + + // Notify this framework that some other framework came into existence. + // This call will be made after the new framework has been linked into the + // `pymb_registry::frameworks` list and before it adds any bindings. + // + // The `pymb_registry::mutex` or GIL will be held when calling this method. + void (*add_foreign_framework)(struct pymb_framework* framework) PYMB_NOEXCEPT; + + // Notify this framework that some other framework is being destroyed. + // This call will be made after the framework has been removed from the + // `pymb_registry::frameworks` list. + // + // This can only occur during interpreter finalization, so no + // synchronization is required. It might occur very late in interpreter + // finalization, such as from a Py_AtExit handler, so it shouldn't + // execute Python code. + void (*remove_foreign_framework)(struct pymb_framework* framework) PYMB_NOEXCEPT; +}; + +/* + * Information about one type binding that belongs to a registered framework. + * + * ### Creating bindings + * + * A framework that binds some type and wants to allow other frameworks to + * work with objects of that type must create a `pymb_binding` structure for + * the type. This can be allocated in any way that the framework prefers (e.g., + * on the heap or within the type object). Any fields without a meaningful + * value must be zero-filled. Once filled out, the binding structure should be + * passed to `pymb_add_binding()`. This will advertise the binding to other + * frameworks' `add_foreign_binding` hooks. It also creates a capsule object + * that points to the `pymb_binding` structure, and stores this capsule in the + * bound type's dict as the attribute "__pymetabind_binding__". + * The intended use of both of these is discussed later on in this comment. + * + * ### Removing bindings + * + * From a user perspective, a binding can be removed for either of two reasons: + * + * - its capsule was destroyed, such as by `del MyType.__pymetabind_binding__` + * - its type object is being finalized + * + * These both result in a call to `pymb_remove_binding()` that begins the + * removal process, but you should not call that function yourself, except + * from a metatype `tp_finalize` as described below. Some time after the call + * to `pymb_remove_binding()`, pymetabind will call the binding's framework's + * `free_local_binding` hook to indicate that it's safe to actually free the + * `pymb_binding` structure. + * + * By default, pymetabind detects the finalization of a binding's type object + * by creating a weakref to the type object with an appropriate callback. This + * works fine, but requires several objects to be allocated, so it is not ideal + * from a memory overhead perspective. If you control the bound type's metatype, + * you can reduce this overhead by modifying the metatype's `tp_finalize` slot + * to call `pymb_remove_binding()`. If you tell pymetabind that you have done + * so, using the `tp_finalize_will_remove` argument to `pymb_add_binding()`, + * then pymetabind won't need to create the weakref and its callback. + * + * ### Removing bindings: the gory details + * + * The implementation of the removal process is somewhat complex in order to + * protect other threads that might be concurrently using a binding in + * free-threaded builds. `pymb_remove_binding()` stops new uses of the binding + * from beginning, by notifying other frameworks' `remove_foreign_binding` + * hooks and changing the binding capsule so `pymb_get_binding()` won't work. + * Existing uses might be ongoing though, so we must wait for them to complete + * before freeing the binding structure. The technique we choose is to wait for + * the next (or current) garbage collection to finish. GC stops all threads + * before it scans the heap. An attached thread state (one that can call + * CPython API functions) can't be stopped without its consent, so GC will + * wait for it to detach. A thread state can only become detached explicitly + * (e.g. Py_BEGIN_ALLOW_THREADS) or in the bytecode interpreter. As long as + * foreign frameworks don't hold `pymb_binding` pointers across calls into + * the bytecode interpreter in places their `remove_foreign_binding` hook + * can't see, this technique avoids use-after-free without introducing any + * contention on a shared atomic in the binding object. + * + * One pleasant aspect of this scheme: due to their use of deferred reference + * counting, type objects in free-threaded Python can only be freed during a + * GC pass. There is even a stop-all-threads (to check for resurrected objects) + * in between when GC executes finalizers and when it actually destroys the + * garbage. This winds up letting us obtain the "wait for next GC pause before + * freeing the binding" behavior very cheaply when the binding is being removed + * due to the deletion of its type. + * + * On non-free-threaded Python builds, none of the above is a concern, and + * `pymb_remove_binding()` can synchronously free the `pymb_binding` structure. + * + * ### Keeping track of other frameworks' bindings + * + * In order to work with Python objects bound by another framework, yours + * must be able to locate a `pymb_binding` structure for that type. It is + * anticipated that most frameworks will maintain their own private + * type-to-binding maps, which they can keep up-to-date via their + * `add_foreign_binding` and `remove_foreign_binding` hooks. It is important + * to think carefully about how to design the synchronization for these maps + * so that lookups do not return pointers to bindings that may have been + * deallocated. The remainder of this section provides some suggestions. + * + * The recommended way to handle synchronization is to protect your type lookup + * map with a readers/writer lock. In your `remove_foreign_binding` hook, + * obtain a write lock, and hold it while removing the corresponding entry from + * the map. Before performing a type lookup, obtain a read lock. If the lookup + * succeeds, you can release the read lock and (due to the two-phase removal + * process described above) continue to safely use the binding for as long as + * your Python thread state remains attached. It is important not to hold the + * read lock while executing arbitrary Python code, since a deadlock would + * result if the binding were removed (requiring a write lock) while the read + * lock were held. Note that `pymb_framework::from_python` for many popular + * frameworks can execute arbitrary Python code to perform implicit conversions. + * + * If you're trying multiple bindings for an operation, one option is to copy + * all their pointers to temporary storage before releasing the read lock. + * (While concurrent updates may modify the data structure, the pymb_binding + * structures it points to will remain valid for long enough.) If you prefer + * to avoid the copy by unlocking for each attempt and then relocking to + * advance to the next binding, be sure to consider the possibility that your + * iterator might have been invalidated due to a concurrent update while you + * weren't holding the lock. + * + * The lock on a single shared type lookup map is a contention bottleneck, + * especially if you don't have a readers/writer lock and wish to get by with + * an ordinary mutex. To improve performance, you can give each thread its + * own lookup map, and require `remove_foreign_binding` to update all of them. + * As long as the per-thread maps are always visited in a consistent order + * when removing a binding, the splitting shouldn't introduce new deadlocks. + * Since each thread can have a separate mutex for its separate map, contention + * occurs only when bindings are being added or removed, which is much less + * common than using them. + * + * ### Using the binding capsule + * + * Each Python type object for which a `pymb_binding` exists will have an + * attribute "__pymetabind_binding__" whose value is a capsule object + * that contains the `pymb_binding` pointer under the name "pymetabind_binding". + * The attribute is set during `pymb_add_binding()`, and is used by + * `pymb_get_binding()` to map a type object to a binding. The capsule allows: + * + * - Determining which framework to call for a foreign `keep_alive` operation + * + * - Locating `pymb_binding` objects for types written in a different language + * than yours (where you can't look up by the `pymb_binding::native_type`), + * so that you can work with their contents using non-Python-specific + * cross-language support + * + * - Extracting the native object from a Python object without being too picky + * about what type it is (risky, but maybe you have out-of-band information + * that shows it's safe) + * + * The preferred mechanism for same-language object access is to maintain a + * hashtable keyed on `pymb_binding::native_type` and look up the binding for + * the type you want/have. Compared to reading the capsule, this better + * supports inheritance, to-Python conversions, and implicit conversions, and + * it's probably also faster depending on how it's implemented. + * + * ### Types with multiple bindings + * + * It is valid for multiple frameworks to claim (in separate bindings) the + * same C/C++ type. This supports cases where a common vocabulary type is + * bound separately in mulitple extensions in the same process. Frameworks + * are encouraged to try all registered bindings for the target type when + * they perform from-Python conversions. + * + * If multiple frameworks claim the same Python type, the last one will + * typically win, since there is only one "__pymetabind_binding__" attribute + * on the type object and a binding is removed when its capsule is no longer + * referenced. If you're trying to do something unusual like wrapping another + * framework's binding to provide additional features, you can stash the + * extra binding(s) under a different attribute name. pymetabind never uses + * the "__pymetabind_binding__" attribute to locate the binding for its own + * purposes; it's used only to fulfill calls to `pymb_get_binding()`. + * + * ### Synchronization + * + * Most fields of this structure are set before it is made visible to other + * threads and then never changed, so they don't need locking to access. The + * `link` and `capsule` are protected by the registry lock. + */ +struct pymb_binding { + // Links to the previous and next bindings in the list of + // `pymb_registry::bindings` + struct pymb_list_node link; + + // The framework that provides this binding + struct pymb_framework* framework; + + // Borrowed reference to the capsule object that refers to this binding. + // Becomes NULL in pymb_remove_binding(). + PyObject* capsule; + + // Python type: you will get an instance of this type from a successful + // call to `framework::from_python()` that passes this binding + PyTypeObject* pytype; + + // Strong reference to a weakref to `pytype`; its callback will prompt + // us to remove the binding. May be NULL if Py_TYPE(pytype)->tp_finalize + // will take care of that. + PyObject* pytype_wr; + + // The native identifier for this type in `framework->abi_lang`, if that is + // a concept that exists in that language. See the documentation of + // `enum pymb_abi_lang` for specific per-language semantics. + const void* native_type; + + // The way that this type would be written in `framework->abi_lang` source + // code, as a NUL-terminated byte string without struct/class/enum words. + // If `framework->abi_lang` uses name mangling, this is the demangled, + // human-readable name. C++ users should note that the result of + // `typeid(x).name()` will need platform-specific alteration to produce one. + // Examples: "Foo", "Bar::Baz", "std::vector >" + const char* source_name; + + // Pointer that is free for use by the framework, e.g., to point to its + // own data about this type. If the framework needs more data, it can + // over-allocate the `pymb_binding` storage and use the space after this. + void* context; +}; + +PYMB_FUNC struct pymb_registry* pymb_get_registry(); +PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, + struct pymb_framework* framework); +PYMB_FUNC void pymb_remove_framework(struct pymb_framework* framework); +PYMB_FUNC void pymb_add_binding(struct pymb_binding* binding, + int tp_finalize_will_remove); +PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding); +PYMB_FUNC void pymb_remove_binding_internal(struct pymb_binding* binding, + int from_capsule_destructor); +PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type); + +#if !defined(PYMB_DECLS_ONLY) + +PYMB_INLINE void pymb_registry_free(struct pymb_registry* registry) { + assert(pymb_list_is_empty(®istry->bindings) && + "some framework was removed before its bindings"); + free(registry->weakref_callback_def); + free(registry); +} + +PYMB_FUNC void pymb_registry_capsule_destructor(PyObject* capsule) { + struct pymb_registry* registry = + (struct pymb_registry*) PyCapsule_GetPointer( + capsule, "pymetabind_registry"); + if (!registry) { + PyErr_WriteUnraisable(capsule); + return; + } + registry->deallocate_when_empty = 1; + if (pymb_list_is_empty(®istry->frameworks)) { + pymb_registry_free(registry); + } +} + +PYMB_FUNC PyObject* pymb_weakref_callback(PyObject* self, PyObject* weakref) { + // self is bound using PyCFunction_New to refer to a capsule that contains + // the binding pointer (not the binding->capsule; this one has no dtor). + // `weakref` is the weakref (to the bound type) that expired. + if (!PyWeakref_CheckRefExact(weakref) || !PyCapsule_CheckExact(self)) { + PyErr_BadArgument(); + return NULL; + } + struct pymb_binding* binding = + (struct pymb_binding*) PyCapsule_GetPointer( + self, "pymetabind_binding"); + if (!binding) { + return NULL; + } + pymb_remove_binding(binding); + Py_RETURN_NONE; +} + +/* + * Locate an existing `pymb_registry`, or create a new one if necessary. + * Returns a pointer to it, or NULL with the CPython error indicator set. + * This must be called from a module initialization function so that the + * import lock can provide mutual exclusion. + */ +PYMB_FUNC struct pymb_registry* pymb_get_registry() { +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + PyObject* dict = PyEval_GetBuiltins(); +#elif PY_VERSION_HEX < 0x03090000 + PyObject* dict = PyInterpreterState_GetDict(_PyInterpreterState_Get()); +#else + PyObject* dict = PyInterpreterState_GetDict(PyInterpreterState_Get()); +#endif + PyObject* key = PyUnicode_FromString("__pymetabind_registry__"); + if (!dict || !key) { + Py_XDECREF(key); + return NULL; + } + PyObject* capsule = PyDict_GetItem(dict, key); + if (capsule) { + Py_DECREF(key); + return (struct pymb_registry*) PyCapsule_GetPointer( + capsule, "pymetabind_registry"); + } + struct pymb_registry* registry; + registry = (struct pymb_registry*) calloc(1, sizeof(*registry)); + if (registry) { + pymb_list_init(®istry->frameworks); + pymb_list_init(®istry->bindings); + registry->deallocate_when_empty = 0; + + // C doesn't allow inline functions to declare static variables, + // so allocate this on the heap + PyMethodDef* def = (PyMethodDef*) calloc(1, sizeof(PyMethodDef)); + if (!def) { + free(registry); + PyErr_NoMemory(); + Py_DECREF(key); + return NULL; + } + def->ml_name = "pymetabind_weakref_callback"; + def->ml_meth = pymb_weakref_callback; + def->ml_flags = METH_O; + def->ml_doc = NULL; + registry->weakref_callback_def = def; + + // Attach a destructor so the registry memory is released at teardown + capsule = PyCapsule_New(registry, "pymetabind_registry", + pymb_registry_capsule_destructor); + if (!capsule) { + free(registry); + registry = NULL; + } else if (PyDict_SetItem(dict, key, capsule) == -1) { + registry = NULL; // will be deallocated by capsule destructor + } + Py_XDECREF(capsule); + } else { + PyErr_NoMemory(); + } + Py_DECREF(key); + return registry; +} + +/* + * Add a new framework to the given registry. Makes calls to + * framework->add_foreign_framework() and framework->add_foreign_binding() + * for each existing framework/binding in the registry. + */ +PYMB_FUNC void pymb_add_framework(struct pymb_registry* registry, + struct pymb_framework* framework) { + // Defensive: ensure hook is clean before first list insertion to avoid UB + framework->link.next = NULL; + framework->link.prev = NULL; + framework->registry = registry; + pymb_lock_registry(registry); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + // Intern `abi_extra` strings so they can be compared by pointer + if (other->abi_extra && framework->abi_extra && + (other->abi_extra == framework->abi_extra || + strcmp(other->abi_extra, framework->abi_extra) == 0)) { + framework->abi_extra = other->abi_extra; + break; + } + } + pymb_list_append(®istry->frameworks, &framework->link); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + if (other != framework) { + other->add_foreign_framework(framework); + framework->add_foreign_framework(other); + } + } + PYMB_LIST_FOREACH(struct pymb_binding*, binding, registry->bindings) { + if (binding->framework != framework) { + framework->add_foreign_binding(binding); + } + } + pymb_unlock_registry(registry); +} + +/* + * Remove a framework from the registry it was added to. + * + * This may only be called during Python interpreter finalization. Rationale: + * other frameworks might be maintaining an entry for the removed one in their + * exception translator lists, and supporting concurrent removal of exception + * translators would add undesirable synchronization overhead to the handling + * of every exception. At finalization time there are no more threads. + * + * Once this function returns, you can free the framework structure. + * + * If a framework never removes itself, it must not claim to be `leak_safe`. + */ +PYMB_FUNC void pymb_remove_framework(struct pymb_framework* framework) { + struct pymb_registry* registry = framework->registry; + + // No need for registry lock/unlock since there are no more threads + pymb_list_unlink(&framework->link); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + other->remove_foreign_framework(framework); + } + + // Destroy registry if capsule is gone and this was the last framework + if (registry->deallocate_when_empty && + pymb_list_is_empty(®istry->frameworks)) { + pymb_registry_free(registry); + } +} + +PYMB_FUNC void pymb_binding_capsule_remove(PyObject* capsule) { + struct pymb_binding* binding = + (struct pymb_binding*) PyCapsule_GetPointer( + capsule, "pymetabind_binding"); + if (!binding) { + PyErr_WriteUnraisable(capsule); + return; + } + pymb_remove_binding_internal(binding, /* from_capsule_destructor */ 1); +} + +/* + * Add a new binding for `binding->framework`. If `tp_finalize_will_remove` is + * nonzero, the caller guarantees that `Py_TYPE(binding->pytype).tp_finalize` + * will call `pymb_remove_binding()`; this saves some allocations compared + * to pymetabind needing to figure out when the type is destroyed on its own. + * See the comment on `pymb_binding` for more details. + */ +PYMB_FUNC void pymb_add_binding(struct pymb_binding* binding, + int tp_finalize_will_remove) { + // Defensive: ensure hook is clean before first list insertion to avoid UB + binding->link.next = NULL; + binding->link.prev = NULL; + + binding->pytype_wr = NULL; + binding->capsule = NULL; + + struct pymb_registry* registry = binding->framework->registry; + if (!tp_finalize_will_remove) { + // Different capsule than the binding->capsule, so that the callback + // doesn't keep the binding alive + PyObject* sub_capsule = PyCapsule_New(binding, "pymetabind_binding", + NULL); + if (!sub_capsule) { + goto error; + } + PyObject* callback = PyCFunction_New(registry->weakref_callback_def, + sub_capsule); + Py_DECREF(sub_capsule); // ownership transferred to callback + if (!callback) { + goto error; + } + binding->pytype_wr = PyWeakref_NewRef((PyObject *) binding->pytype, + callback); + Py_DECREF(callback); // ownership transferred to weakref + if (!binding->pytype_wr) { + goto error; + } + } else { +#if defined(Py_GIL_DISABLED) + // No callback needed in this case, but we still do need the weakref + // so that pymb_remove_binding() can tell if the type is being + // finalized or not. + binding->pytype_wr = PyWeakref_NewRef((PyObject *) binding->pytype, + NULL); + if (!binding->pytype_wr) { + goto error; + } +#endif + } + + binding->capsule = PyCapsule_New(binding, "pymetabind_binding", + pymb_binding_capsule_remove); + if (!binding->capsule) { + goto error; + } + if (PyObject_SetAttrString((PyObject *) binding->pytype, + "__pymetabind_binding__", + binding->capsule) != 0) { + Py_CLEAR(binding->capsule); + goto error; + } + Py_DECREF(binding->capsule); // keep only a borrowed reference + + pymb_lock_registry(registry); + pymb_list_append(®istry->bindings, &binding->link); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + if (other != binding->framework) { + other->add_foreign_binding(binding); + } + } + pymb_unlock_registry(registry); + return; + + error: + PyErr_WriteUnraisable((PyObject *) binding->pytype); + Py_XDECREF(binding->pytype_wr); + binding->framework->free_local_binding(binding); +} + +#if defined(Py_GIL_DISABLED) +PYMB_FUNC void pymb_binding_capsule_destroy(PyObject* capsule) { + struct pymb_binding* binding = + (struct pymb_binding*) PyCapsule_GetPointer( + capsule, "pymetabind_binding"); + if (!binding) { + PyErr_WriteUnraisable(capsule); + return; + } + Py_CLEAR(binding->pytype_wr); + binding->framework->free_local_binding(binding); +} +#endif + +/* + * Remove a binding from the registry it was added to. Don't call this yourself, + * except from the tp_finalize slot of a binding's type's metatype. + * The user-servicable way to remove a binding from a still-alive type is to + * delete the capsule. The binding structure will eventually be freed by calling + * `binding->framework->free_local_binding(binding)`. + */ +PYMB_FUNC void pymb_remove_binding(struct pymb_binding* binding) { + pymb_remove_binding_internal(binding, 0); +} + +PYMB_FUNC void pymb_remove_binding_internal(struct pymb_binding* binding, + int from_capsule_destructor) { + struct pymb_registry* registry = binding->framework->registry; + + // Since we need to obtain it anyway, use the registry lock to serialize + // concurrent attempts to remove the same binding + pymb_lock_registry(registry); + if (!binding->capsule) { + // Binding was concurrently removed from multiple places; the first + // one to get the registry lock wins. + pymb_unlock_registry(registry); + return; + } + +#if defined(Py_GIL_DISABLED) + // Determine if binding->pytype is still fully alive (not yet started + // finalizing). If so, it can't die until the next GC cycle, so freeing + // the binding at the next GC is safe. + PyObject* pytype_strong = NULL; + if (PyWeakref_GetRef(binding->pytype_wr, &pytype_strong) == -1) { + // If something's wrong with the weakref, leave pytype_strong set to + // NULL in order to conservatively assume the type is finalizing. + // This will leak the binding struct until the type object is destroyed. + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } +#endif + + // Clear the existing capsule's destructor so we don't have to worry about + // it firing after the pymb_binding struct has actually been freed. + // Note we can safely assume the capsule hasn't been freed yet. + // (Proof: Its destructor calls this function, which cannot complete until + // it acquires the lock we currently hold. If the destructor completed + // already, we would have bailed out above upon noticing capsule was + // already NULL.) + if (!from_capsule_destructor && + PyCapsule_SetDestructor(binding->capsule, NULL) != 0) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } + + // Mark this binding as being in the process of being destroyed. + binding->capsule = NULL; + + // If weakref hasn't fired yet, we don't need it anymore. Destroying it + // ensures it won't fire after the binding struct has been freed. + Py_CLEAR(binding->pytype_wr); + + pymb_list_unlink(&binding->link); + binding->framework->remove_local_binding(binding); + PYMB_LIST_FOREACH(struct pymb_framework*, other, registry->frameworks) { + if (other != binding->framework) { + other->remove_foreign_binding(binding); + } + } + pymb_unlock_registry(registry); + +#if !defined(Py_GIL_DISABLED) + // On GIL builds, there's no need to delay deallocation + binding->framework->free_local_binding(binding); +#else + // Create a new capsule to manage the actual freeing + PyObject* capsule_destroy = PyCapsule_New(binding, + "pymetabind_binding", + pymb_binding_capsule_destroy); + if (!capsule_destroy) { + // Just leak the binding if we can't set up the capsule + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } else if (pytype_strong) { + // Type still alive -> embed the capsule in a cycle so it lasts until + // next GC. (The type will live at least that long.) + PyObject* list = PyList_New(2); + if (!list) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + // leak the capsule and therefore the binding + } else { + PyList_SetItem(list, 0, capsule_destroy); + PyList_SetItem(list, 1, list); + // list is now referenced only by itself and will be GCable + } + } else { + // Type is dying -> destroy the capsule when the type is destroyed. + // Since the type's weakrefs were already cleared, any weakref we add + // now won't fire until the type's tp_dealloc. We reuse our existing + // weakref callback for convenience; the call that it makes to + // pymb_remove_binding() will be a no-op, but after it fires, + // the capsule destructor will do the freeing we desire. + PyObject* callback = PyCFunction_New(registry->weakref_callback_def, + capsule_destroy); + if (!callback) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + // leak the capsule and therefore the binding + } else { + Py_DECREF(capsule_destroy); // ownership transferred to callback + binding->pytype_wr = PyWeakref_NewRef((PyObject *) binding->pytype, + callback); + Py_DECREF(callback); // ownership transferred to weakref + if (!binding->pytype_wr) { + PyErr_WriteUnraisable((PyObject *) binding->pytype); + } + } + } + Py_XDECREF(pytype_strong); +#endif +} + +/* + * Return a pointer to a pymb_binding for the Python type `type`, or NULL if + * none exists. + */ +PYMB_FUNC struct pymb_binding* pymb_get_binding(PyObject* type) { + PyObject* capsule = PyObject_GetAttrString(type, "__pymetabind_binding__"); + if (capsule == NULL) { + PyErr_Clear(); + return NULL; + } + void* binding = PyCapsule_GetPointer(capsule, "pymetabind_binding"); + Py_DECREF(capsule); + if (!binding) { + PyErr_Clear(); + } + return (struct pymb_binding*) binding; +} + +#endif /* defined(PYMB_DECLS_ONLY) */ + +#if defined(__cplusplus) +} +#endif diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index cd7e87f845..bd230ec6f2 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -226,6 +226,7 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { } else { internals.registered_types_cpp.erase(tindex); } + get_interop_internals().copy_move_ctors.erase(tindex); internals.registered_types_py.erase(tinfo->type); // Actually just `std::erase_if`, but that's only available in C++20 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 22dfc62b4c..6196b5ad0d 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -214,6 +214,12 @@ # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif +#if defined(_MSC_VER) +# define PYBIND11_COMPAT_STRDUP _strdup +#else +# define PYBIND11_COMPAT_STRDUP strdup +#endif + #include #include #include @@ -268,6 +274,21 @@ # endif #endif +// 3.14 Compatibility +#if !defined(Py_GIL_DISABLED) +inline bool is_uniquely_referenced(PyObject *obj) { return Py_REFCNT(obj) == 1; } +#elif 0x030E0000 <= PY_VERSION_HEX +inline bool is_uniquely_referenced(PyObject *obj) { + return PyUnstable_Object_IsUniquelyReferenced(obj); +} +#else // backport for 3.13 +inline bool is_uniquely_referenced(PyObject *obj) { + return _Py_IsOwnedByCurrentThread(obj) + && _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local) == 1 + && _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared) == 0; +} +#endif + // 3.13 Compatibility #if 0x030D0000 <= PY_VERSION_HEX # define PYBIND11_TYPE_IS_TYPE_HINT "typing.TypeIs" @@ -348,6 +369,7 @@ #define PYBIND11_ENSURE_INTERNALS_READY \ { \ pybind11::detail::get_internals_pp_manager().unref(); \ + pybind11::detail::get_interop_internals_pp_manager().unref(); \ pybind11::detail::get_internals(); \ } @@ -1344,5 +1366,15 @@ constexpr # define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET #endif +#if defined(PY_BIG_ENDIAN) +# define PYBIND11_BIG_ENDIAN PY_BIG_ENDIAN +#else // pypy doesn't define PY_BIG_ENDIAN +# if defined(_MSC_VER) +# define PYBIND11_BIG_ENDIAN 0 // All Windows platforms are little-endian +# else // GCC and Clang define the following macros +# define PYBIND11_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# endif +#endif + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/foreign.h b/include/pybind11/detail/foreign.h new file mode 100644 index 0000000000..f76aa032c0 --- /dev/null +++ b/include/pybind11/detail/foreign.h @@ -0,0 +1,818 @@ +/* + pybind11/detail/foreign.h: Interoperability with other binding frameworks + + Copyright (c) 2025 Hudson River Trading + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include + +#include "common.h" +#include "internals.h" +#include "type_caster_base.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// pybind11 exception translator that tries all known foreign ones +PYBIND11_NOINLINE void foreign_exception_translator(std::exception_ptr p) { + auto &interop_internals = get_interop_internals(); + for (pymb_framework *fw : interop_internals.exc_frameworks) { + if (fw->translate_exception(&p) != 0) { + return; + } + } + std::rethrow_exception(p); +} + +// When learning about a new foreign type, should we automatically use it? +inline bool should_autoimport_foreign(interop_internals &interop_internals, + pymb_binding *binding) { + return interop_internals.import_all && binding->framework->abi_lang == pymb_abi_lang_cpp + && binding->framework->abi_extra == interop_internals.self->abi_extra; +} + +// Determine whether a pybind11 type is module-local from a different module +inline bool is_local_to_other_module(type_info *ti) { + return ti->module_local_load != nullptr + && ti->module_local_load != &type_caster_generic::local_load; +} + +// Add the given `binding` to our type maps so that we can use it to satisfy +// from- and to-Python requests for the given C++ type +inline void import_foreign_binding(pymb_binding *binding, const std::type_info *cpptype) noexcept { + // Caller must hold the internals lock + auto &interop_internals = get_interop_internals(); + interop_internals.imported_any = true; + auto &lst = interop_internals.bindings[*cpptype]; + for (pymb_binding *existing : lst) { + if (existing == binding) { + return; // already imported + } + } + ++interop_internals.bindings_update_count; + lst.append(binding); +} + +// Callback functions for other frameworks to operate on our objects +// or tell us about theirs + +inline void *interop_cb_from_python(pymb_binding *binding, + PyObject *pyobj, + uint8_t convert, + void (*keep_referenced)(void *ctx, PyObject *obj), + void *keep_referenced_ctx) noexcept { + if (binding->context == nullptr) { + // This is a native enum type. We can only return a pointer to the C++ + // enum if we're able to allocate a temporary. + handle pytype((PyObject *) binding->pytype); + if (!keep_referenced || !isinstance(pyobj, pytype)) { + return nullptr; + } + try { + auto cap + = reinterpret_borrow(pytype.attr(native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + auto value = handle(pyobj).attr("value"); + uint64_t ival = 0; + if (info->is_signed && handle(value) < int_(0)) { + ival = (uint64_t) cast(value); + } else { + ival = cast(value); + } + bytes holder{reinterpret_cast(&ival) + + PYBIND11_BIG_ENDIAN * size_t(8 - info->size_bytes), + info->size_bytes}; + keep_referenced(keep_referenced_ctx, holder.ptr()); + return PyBytes_AsString(holder.ptr()); + } catch (error_already_set &exc) { + exc.discard_as_unraisable("Error converting native enum from Python"); + return nullptr; + } + } + +#if defined(PYBIND11_HAS_OPTIONAL) + using maybe_life_support = std::optional; +#else + struct maybe_life_support { + union { + loader_life_support supp; + }; + bool engaged = false; + + maybe_life_support() {} + maybe_life_support(maybe_life_support &) = delete; + loader_life_support *operator->() { return &supp; } + void emplace() { + new (&supp) loader_life_support(); + engaged = true; + } + ~maybe_life_support() { + if (engaged) { + supp.~loader_life_support(); + } + } + }; +#endif + maybe_life_support holder; + if (keep_referenced) { + holder.emplace(); + } + type_caster_generic caster{static_cast(binding->context)}; + void *ret = nullptr; + try { + if (caster.load_impl(pyobj, + convert != 0, + /* foreign_ok */ false)) { + ret = caster.value; + } + } catch (...) { + translate_exception(std::current_exception()); + PyErr_WriteUnraisable(pyobj); + } + if (keep_referenced) { + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + for (PyObject *item : holder->list_patients()) { + keep_referenced(keep_referenced_ctx, item); + } + } + return ret; +} + +// This wraps the call to type_info::init_instance() in some cases when casting +// a pybind11-bound object to Python on behalf of a foreign framework. It +// inhibits registration of the new instance so that interop_cb_keep_alive() +// can fix up the holder before other threads start using the new instance. +inline void init_instance_unregistered(instance *inst, const void *holder) { + assert(holder == nullptr && !inst->owned); + (void) holder; // avoid unused warning if compiled without asserts + value_and_holder v_h = *values_and_holders(inst).begin(); + + // If using smart_holder, force creation of a shared_ptr that has a + // guarded_delete deleter, so that we can modify it in + // interop_cb_keep_alive(). We can't create it there because it needs to be + // created in the same DSO as it's accessed in; init_instance is in that + // DSO, but this function might not be. + if (v_h.type->holder_enum_v == holder_enum_t::smart_holder) { + inst->owned = true; + } + + // Pretend it's already registered so that init_instance doesn't try again + v_h.set_instance_registered(true); + + // Undo our shenanigans even if init_instance raises an exception + struct guard { + value_and_holder &v_h; + ~guard() noexcept { + v_h.set_instance_registered(false); + if (v_h.type->holder_enum_v == holder_enum_t::smart_holder) { + v_h.inst->owned = false; + auto &h = v_h.holder(); + h.vptr_is_using_std_default_delete = true; + h.reset_vptr_deleter_armed_flag(v_h.type->get_memory_guarded_delete, + /* armed_flag */ false); + } + } + } guard{v_h}; + v_h.type->init_instance(inst, nullptr); +} + +inline PyObject *interop_cb_to_python(pymb_binding *binding, + void *cobj, + enum pymb_rv_policy rvp_, + pymb_to_python_feedback *feedback) noexcept { + feedback->relocate = 0; // we don't support relocation + feedback->is_new = 0; // unless overridden below + + if (cobj == nullptr) { + return none().release().ptr(); + } + + if (!binding->context) { + // Native enum type + try { + handle pytype((PyObject *) binding->pytype); + auto cap + = reinterpret_borrow(pytype.attr(native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + uint64_t key = 0; + switch (info->size_bytes) { + case 1: + key = *(uint8_t *) cobj; + break; + case 2: + key = *(uint16_t *) cobj; + break; + case 4: + key = *(uint32_t *) cobj; + break; + case 8: + key = *(uint64_t *) cobj; + break; + default: + return nullptr; + } + if (rvp_ == pymb_rv_policy_take_ownership) { + ::operator delete(cobj); + } + if (info->is_signed) { + auto ikey = (int64_t) key; + if (info->size_bytes < 8) { + // sign extend + ikey <<= (64 - (info->size_bytes * 8)); + ikey >>= (64 - (info->size_bytes * 8)); + } + return pytype(ikey).release().ptr(); + } + return pytype(key).release().ptr(); + } catch (error_already_set &exc) { + exc.restore(); + return nullptr; + } + } + + const auto *ti = static_cast(binding->context); + return_value_policy rvp = return_value_policy::automatic; + bool inhibit_registration = false; + + switch (rvp_) { + case pymb_rv_policy_take_ownership: + case pymb_rv_policy_copy: + case pymb_rv_policy_move: + case pymb_rv_policy_reference: + // These have the same values and semantics as our own policies + rvp = (return_value_policy) rvp_; + break; + case pymb_rv_policy_share_ownership: + rvp = return_value_policy::reference; + inhibit_registration = true; + break; + case pymb_rv_policy_none: + break; + } + if (rvp == return_value_policy::automatic) { + // Specified rvp was none, or was something unrecognized so we should + // be conservative and treat it like none. + return find_registered_python_instance(cobj, ti).ptr(); + } + + copy_or_move_ctor copy_ctor = nullptr, move_ctor = nullptr; + if (rvp == return_value_policy::copy || rvp == return_value_policy::move) { + with_internals([&](internals &) { + auto &interop_internals = get_interop_internals(); + auto it = interop_internals.copy_move_ctors.find(*ti->cpptype); + if (it != interop_internals.copy_move_ctors.end()) { + std::tie(copy_ctor, move_ctor) = it->second; + } + }); + } + + try { + type_caster_generic::cast_sources srcs{cobj, ti}; + if (inhibit_registration) { + srcs.init_instance = init_instance_unregistered; + } + handle ret = type_caster_generic::cast(srcs, rvp, {}, copy_ctor, move_ctor); + feedback->is_new = uint8_t(srcs.is_new); + return ret.ptr(); + } catch (...) { + translate_exception(std::current_exception()); + return nullptr; + } +} + +inline int interop_cb_keep_alive(PyObject *nurse, void *payload, void (*cb)(void *)) noexcept { + try { + do { // single-iteration loop to reduce nesting level + if (!is_uniquely_referenced(nurse)) { + break; + } + // See if we can install this as a shared_ptr deleter rather than + // a keep_alive, since the very first keep_alive for a new object + // might be to let it carry shared_ptr ownership. This helps + // a shared_ptr returned from a foreign binding be acceptable + // as a shared_ptr argument to a pybind11-bound function. + values_and_holders vhs{nurse}; + if (vhs.size() != 1) { + break; + } + value_and_holder v_h = *vhs.begin(); + if (v_h.instance_registered()) { + break; + } + auto cb_to_use = cb ? cb : (decltype(cb)) Py_DecRef; + bool success = false; + if (v_h.type->holder_enum_v == holder_enum_t::std_shared_ptr + && !v_h.holder_constructed()) { + // Create a shared_ptr whose destruction will perform the action + std::shared_ptr owner(payload, cb_to_use); + // Use the aliasing constructor to make its get() return the right thing + // NB: this constructor accepts an rvalue reference only in C++20 + new (std::addressof(v_h.holder>())) + // NOLINTNEXTLINE(performance-move-const-arg) + std::shared_ptr(std::move(owner), v_h.value_ptr()); + v_h.set_holder_constructed(); + success = true; + } else if (v_h.type->holder_enum_v == holder_enum_t::smart_holder + && v_h.holder_constructed() && !v_h.inst->owned) { + auto &h = v_h.holder(); + auto *gd = v_h.type->get_memory_guarded_delete(h.vptr); + if (gd && !gd->armed_flag) { + gd->del_fun = [=](void *) { cb_to_use(payload); }; + gd->use_del_fun = true; + gd->armed_flag = true; + success = true; + } + } + register_instance(v_h.inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(true); + if (success) { + return 1; + } + } while (false); + + if (!cb) { + keep_alive_impl(nurse, static_cast(payload)); + } else { + capsule patient{payload, cb}; + keep_alive_impl(nurse, patient); + } + return 1; + } catch (...) { + translate_exception(std::current_exception()); + PyErr_WriteUnraisable(nurse); + return 0; + } +} + +inline int interop_cb_translate_exception(void *eptr) noexcept { + return with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list & /*local_exception_translators*/) { + // Ignore local exception translators. We're being called to translate + // an exception that was raised from a different framework, thus a + // different extension module, so nothing local to us will apply. + // Try global translators, except the last one or two. + std::exception_ptr &e = *(std::exception_ptr *) eptr; + auto it = exception_translators.begin(); + auto leader = it; + // - The last one is the default translator. It translates + // standard exceptions, which we should leave up to the + // framework that bound a function. + ++leader; + // - If we've installed the foreign_exception_translator hook + // (for pybind11 functions to be able to translate other + // frameworks' exceptions), it's the second-last one and should + // be skipped too. We don't want mutual recursion between + // different frameworks' translators. + if (!get_interop_internals().exc_frameworks.empty()) { + ++leader; + } + + for (; leader != exception_translators.end(); ++it, ++leader) { + try { + (*it)(e); + return 1; + } catch (...) { + e = std::current_exception(); + } + } + + // Try the part of the default translator that is pybind11-specific + try { + std::rethrow_exception(e); + } catch (error_already_set &err) { + handle_nested_exception(err, e); + err.restore(); + return 1; + } catch (const builtin_exception &err) { + // Could not use template since it's an abstract class. + if (const auto *nep + = dynamic_cast(std::addressof(err))) { + handle_nested_exception(*nep, e); + } + err.set_error(); + return 1; + } catch (...) { + e = std::current_exception(); + } + return 0; + }); +} + +inline void interop_cb_remove_local_binding(pymb_binding *binding) noexcept { + with_internals([&](internals &) { + auto &interop_internals = get_interop_internals(); + const auto *cpptype = (const std::type_info *) binding->native_type; + auto it = interop_internals.bindings.find(*cpptype); + if (it != interop_internals.bindings.end() && it->second.erase(binding)) { + ++interop_internals.bindings_update_count; + if (it->second.empty()) { + interop_internals.bindings.erase(it); + } + } + }); +} + +inline void interop_cb_free_local_binding(pymb_binding *binding) noexcept { + free(const_cast(binding->source_name)); + delete binding; +} + +inline void interop_cb_add_foreign_binding(pymb_binding *binding) noexcept { + with_internals([&](internals &) { + auto &interop_internals = get_interop_internals(); + if (should_autoimport_foreign(interop_internals, binding)) { + import_foreign_binding(binding, (const std::type_info *) binding->native_type); + } + }); +} + +inline void interop_cb_remove_foreign_binding(pymb_binding *binding) noexcept { + with_internals([&](internals &) { + auto &interop_internals = get_interop_internals(); + auto remove_from_type = [&](const std::type_info *type) { + auto it = interop_internals.bindings.find(*type); + if (it != interop_internals.bindings.end() && it->second.erase(binding)) { + ++interop_internals.bindings_update_count; + if (it->second.empty()) { + interop_internals.bindings.erase(it); + } + } + }; + bool should_remove_auto = should_autoimport_foreign(interop_internals, binding); + auto it = interop_internals.manual_imports.find(binding); + if (it != interop_internals.manual_imports.end()) { + remove_from_type(it->second); + should_remove_auto &= (it->second != binding->native_type); + interop_internals.manual_imports.erase(it); + } + if (should_remove_auto) { + remove_from_type((const std::type_info *) binding->native_type); + } + }); +} + +inline void interop_cb_add_foreign_framework(pymb_framework *framework) noexcept { + if (framework->translate_exception) { + with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &) { + auto &interop_internals = get_interop_internals(); + if (interop_internals.exc_frameworks.empty()) { + // First foreign framework with an exception translator. + // Add our `foreign_exception_translator` wrapper in the + // 2nd-last position (last is the default exception + // translator). + auto leader = exception_translators.begin(); + auto trailer = exception_translators.before_begin(); + while (++leader != exception_translators.end()) { + ++trailer; + } + exception_translators.insert_after(trailer, foreign_exception_translator); + } + // Add the new framework at the end of the list + auto it = interop_internals.exc_frameworks.before_begin(); + while (std::next(it) != interop_internals.exc_frameworks.end()) { + ++it; + } + interop_internals.exc_frameworks.insert_after(it, framework); + }); + } +} + +inline void interop_cb_remove_foreign_framework(pymb_framework *framework) noexcept { + // No need for locking; the interpreter is already finalizing + // at this point (and might be already finalized, so we can't do any + // Python API calls) + if (framework->translate_exception) { + get_interop_internals().exc_frameworks.remove(framework); + // No need to bother removing the foreign_exception_translator if + // this was the last of the exc_frameworks. In the unlikely event + // that something needs an exception translated during finalization, + // it will work fine with an empty exc_frameworks list. + } +} + +// (end of callbacks) + +// Advertise our existence, and the above callbacks, to other frameworks +PYBIND11_NOINLINE bool interop_internals::initialize() { + pymb_registry *registry = nullptr; + bool inited_by_us = with_internals([&](internals &) { + if (self) { + return false; + } + registry = pymb_get_registry(); + if (!registry) { + throw error_already_set(); + } + + self.reset(new pymb_framework{}); + self->name = "pybind11 " PYBIND11_ABI_TAG; + self->flags = 0; + self->abi_lang = pymb_abi_lang_cpp; + self->abi_extra = PYBIND11_PLATFORM_ABI_ID; + self->from_python = interop_cb_from_python; + self->to_python = interop_cb_to_python; + self->keep_alive = interop_cb_keep_alive; + self->translate_exception = interop_cb_translate_exception; + self->remove_local_binding = interop_cb_remove_local_binding; + self->free_local_binding = interop_cb_free_local_binding; + self->add_foreign_binding = interop_cb_add_foreign_binding; + self->remove_foreign_binding = interop_cb_remove_foreign_binding; + self->add_foreign_framework = interop_cb_add_foreign_framework; + self->remove_foreign_framework = interop_cb_remove_foreign_framework; + return true; + }); + if (inited_by_us) { + // Unlock internals before calling add_framework, so that the callbacks + // (interop_cb_add_foreign_binding, etc) can safely re-lock it. + pymb_add_framework(registry, self.get()); + } + return inited_by_us; +} + +inline interop_internals::~interop_internals() { + if (self && bindings.empty()) { + pymb_remove_framework(self.get()); + } +} + +// Learn to satisfy from- and to-Python requests for `cpptype` using the +// foreign binding provided by the given `pytype`. If cpptype is nullptr, infer +// the C++ type by looking at the binding, and require that its ABI match ours. +// Throws an exception on failure. Caller must hold the internals lock and have +// already called interop_internals.initialize_if_needed(). +PYBIND11_NOINLINE void import_for_interop(handle pytype, const std::type_info *cpptype) { + auto &interop_internals = get_interop_internals(); + pymb_binding *binding = pymb_get_binding(pytype.ptr()); + if (!binding) { + pybind11_fail("pybind11::import_for_interop(): type does not define " + "a __pymetabind_binding__"); + } + if (binding->pytype != (PyTypeObject *) pytype.ptr()) { + pybind11_fail("pybind11::import_for_interop(): the binding associated " + "with the type you specified is for a different type; " + "pass the type object that was created by the other " + "framework, not its subclass"); + } + if (binding->framework == interop_internals.self.get()) { + // Can't call get_type_info() because it would lock internals and + // they're already locked + auto &internals = get_internals(); + auto it = internals.registered_types_py.find(binding->pytype); + if (it != internals.registered_types_py.end() && it->second.size() == 1 + && is_local_to_other_module(*it->second.begin())) { + // Allow importing module-local types from other pybind11 modules, + // even if they're ABI-compatible with us and thus use the same + // pymb_framework. The import is not doing much here; the export + // alone would put the binding in interop_internals where we can + // see it. + } else { + pybind11_fail("pybind11::import_for_interop(): type is not foreign"); + } + } + if (!cpptype) { + if (binding->framework->abi_lang != pymb_abi_lang_cpp) { + pybind11_fail("pybind11::import_for_interop(): type is not " + "written in C++, so you must specify a C++ type"); + } + if (binding->framework->abi_extra != interop_internals.self->abi_extra) { + pybind11_fail("pybind11::import_for_interop(): type has " + "incompatible C++ ABI with this module"); + } + cpptype = (const std::type_info *) binding->native_type; + } + + auto result = interop_internals.manual_imports.emplace(binding, cpptype); + if (!result.second) { + const auto *existing = (const std::type_info *) result.first->second; + if (existing != cpptype && *existing != *cpptype) { + pybind11_fail("pybind11::import_for_interop(): type was " + "already imported as a different C++ type"); + } + } + import_foreign_binding(binding, cpptype); +} + +// Call `import_foreign_binding()` for every ABI-compatible type provided by +// other C++ binding frameworks used by extension modules loaded in this +// interpreter, both those that exist now and those bound in the future. +PYBIND11_NOINLINE void interop_enable_import_all() { + auto &interop_internals = get_interop_internals(); + bool proceed = with_internals([&](internals &) { + if (interop_internals.import_all) { + return false; + } + interop_internals.import_all = true; + return true; + }); + if (!proceed) { + return; + } + if (interop_internals.initialize_if_needed()) { + // pymb_add_framework tells us about every existing type when we + // register, so if we register with import enabled, we're done + return; + } + // If we enable import after registering, we have to iterate over the + // list of types ourselves. Do this without the internals lock held so + // we can reuse the pymb callback functions. interop_internals registry + + // self never change once they're non-null, so we can access them + // without locking here. + struct pymb_registry *registry = interop_internals.self->registry; + pymb_lock_registry(registry); + // NOLINTNEXTLINE(modernize-use-auto) + PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { + if (binding->framework != interop_internals.self.get()) { + interop_cb_add_foreign_binding(binding); + } + } + pymb_unlock_registry(registry); +} + +// Expose hooks for other frameworks to use to work with the given pybind11 +// type object. `ti` may be nullptr if exporting a native enum. +// Caller must hold the internals lock and have already called +// interop_internals.initialize_if_needed(). +PYBIND11_NOINLINE void +export_for_interop(const std::type_info *cpptype, PyTypeObject *pytype, type_info *ti) { + auto &interop_internals = get_interop_internals(); + auto &lst = interop_internals.bindings[*cpptype]; + for (pymb_binding *existing : lst) { + if (existing->framework == interop_internals.self.get() && existing->pytype == pytype) { + return; // already imported + } + } + + auto *binding = new pymb_binding{}; + binding->framework = interop_internals.self.get(); + binding->pytype = pytype; + binding->native_type = cpptype; + binding->source_name = PYBIND11_COMPAT_STRDUP(clean_type_id(cpptype->name()).c_str()); + binding->context = ti; + + ++interop_internals.bindings_update_count; + lst.append(binding); + pymb_add_binding(binding, /* tp_finalize_will_remove */ 0); +} + +// Call `export_type_to_foreign()` for each type that currently exists in our +// internals structure and each type created in the future. +PYBIND11_NOINLINE void interop_enable_export_all() { + auto &interop_internals = get_interop_internals(); + bool proceed = with_internals([&](internals &) { + if (interop_internals.export_all) { + return false; + } + interop_internals.export_all = true; + interop_internals.export_for_interop = &detail::export_for_interop; + return true; + }); + if (!proceed) { + return; + } + interop_internals.initialize_if_needed(); + with_internals([&](internals &internals) { + for (const auto &entry : internals.registered_types_cpp) { + auto *ti = entry.second; + detail::export_for_interop(ti->cpptype, ti->type, ti); + } + for (const auto &entry : internals.native_enum_type_map) { + try { + auto cap = reinterpret_borrow( + handle(entry.second).attr(native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + detail::export_for_interop(info->cpptype, (PyTypeObject *) entry.second, nullptr); + } catch (error_already_set &) { // NOLINT(bugprone-empty-catch) + // Ignore native enums without a __pybind11_enum__ capsule; + // they might be from an older version of pybind11 + } + } + }); +} + +// Invoke `attempt(closure, binding)` for each foreign binding `binding` +// that claims `type` and was not supplied by us, until one of them returns +// non-null. Return that first non-null value, or null if all attempts failed. +PYBIND11_NOINLINE void *try_foreign_bindings(const std::type_info *type, + void *(*attempt)(void *closure, + pymb_binding *binding), + void *closure) { + auto &internals = get_internals(); + auto &interop_internals = get_interop_internals(); + uint32_t update_count = interop_internals.bindings_update_count; + + do { + PYBIND11_LOCK_INTERNALS(internals); + (void) internals; // suppress unused warning on non-ft builds + auto it = interop_internals.bindings.find(*type); + if (it == interop_internals.bindings.end()) { + return nullptr; + } + for (pymb_binding *binding : it->second) { + if (binding->framework == interop_internals.self.get() + && (!binding->context + || !is_local_to_other_module((type_info *) binding->context))) { + // Don't try to use our own types, unless they're module-local + // to some other module and this is the only way we'd see them. + // (The module-local escape hatch is only relevant for + // to-Python conversions; from-Python won't try foreign if it + // sees the capsule for other-module-local.) + continue; + } + +#ifdef Py_GIL_DISABLED + // attempt() might execute Python code; drop the internals lock + // to avoid a deadlock + lock.unlock(); +#endif + void *result = attempt(closure, binding); + if (result) { + return result; + } +#ifdef Py_GIL_DISABLED + lock.lock(); +#endif + // Make sure our iterator wasn't invalidated by something that + // was done within attempt(), or concurrently during attempt() + // while we didn't hold the internals lock + if (interop_internals.bindings_update_count != update_count) { + // Concurrent update occurred; stop iterating + break; + } + } + if (interop_internals.bindings_update_count != update_count) { + // We broke out early due to a concurrent update. Retry from the top. + update_count = interop_internals.bindings_update_count; + continue; + } + return nullptr; + } while (true); +} + +PYBIND11_NAMESPACE_END(detail) + +inline void interoperate_by_default(bool export_all = true, bool import_all = true) { + auto &interop_internals = detail::get_interop_internals(); + if (import_all && !interop_internals.import_all) { + detail::interop_enable_import_all(); + } + if (export_all && !interop_internals.export_all) { + detail::interop_enable_export_all(); + } +} + +template +inline void import_for_interop(handle pytype) { + if (!PyType_Check(pytype.ptr())) { + pybind11_fail("pybind11::import_for_interop(): expected a type object"); + } + const std::type_info *cpptype = std::is_void::value ? nullptr : &typeid(T); + auto &interop_internals = detail::get_interop_internals(); + interop_internals.initialize_if_needed(); + detail::with_internals( + [&](detail::internals &) { detail::import_for_interop(pytype, cpptype); }); +} + +inline void export_for_interop(handle ty) { + if (!PyType_Check(ty.ptr())) { + pybind11_fail("pybind11::export_for_interop(): expected a type object"); + } + auto &interop_internals = detail::get_interop_internals(); + interop_internals.initialize_if_needed(); + detail::type_info *ti = detail::get_type_info((PyTypeObject *) ty.ptr()); + if (ti) { + detail::with_internals( + [&](detail::internals &) { detail::export_for_interop(ti->cpptype, ti->type, ti); }); + return; + } + // Not a class_; maybe it's a native_enum? + try { + auto cap + = reinterpret_borrow(ty.attr(detail::native_enum_info::attribute_name())); + auto *info = cap.get_pointer(); + bool ours = detail::with_internals([&](detail::internals &internals) { + auto it = internals.native_enum_type_map.find(*info->cpptype); + if (it != internals.native_enum_type_map.end() && it->second == ty.ptr()) { + detail::export_for_interop(info->cpptype, (PyTypeObject *) ty.ptr(), nullptr); + return true; + } + return false; + }); + if (ours) { + return; + } + } catch (error_already_set &) { // NOLINT(bugprone-empty-catch) + // Could be an older native enum without __pybind11_enum__ capsule + } + pybind11_fail("pybind11::export_for_interop: not a " + "pybind11 class or enum bound in this domain"); +} + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d23ee6ec91..92dc6850c7 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -24,6 +24,10 @@ #include #include +struct pymb_binding; +struct pymb_framework; +struct pymb_registry; + /// Tracks the `internals` and `type_info` ABI version independent of the main library version. /// /// Some portions of the code use an ABI that is conditional depending on this @@ -39,7 +43,8 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -// REMINDER for next version bump: remove loader_life_support_tls +// REMINDER for next version bump: remove loader_life_support_tls + +// merge interop_internals into internals # define PYBIND11_INTERNALS_VERSION 11 #endif @@ -295,6 +300,147 @@ struct internals { ~internals() = default; }; +using copy_or_move_ctor = void *(*) (const void *); + +// Information to support working with types from other binding frameworks or +// other ABI versions of pybind11, and sharing our types with them. This should +// logically be part of `internals`, but we want to support it without an ABI +// version bump. If it gets incorpoprated into `internals` in a later ABI +// version, it would be good for performance to also add a flag to `type_info` +// indicating whether any foreign bindings are also known for its C++ type; +// that way we can avoid an extra lookup when conversion to a native type fails. +struct interop_internals { + // A vector optimized for the case of storing one element. + class binding_list { + public: + binding_list() : single(nullptr) {} + binding_list(const binding_list &) = delete; + binding_list(binding_list &&other) noexcept : vec(other.vec) { other.vec = 0; } + ~binding_list() { + if (is_vec()) { + delete as_vec(); + } + } + + bool empty() const { return single == nullptr; } + pymb_binding **begin() { + return empty() ? nullptr : is_vec() ? as_vec()->data() : &single; + } + pymb_binding **end() { + return empty() ? nullptr + : is_vec() ? as_vec()->data() + as_vec()->size() + : &single + 1; + } + void append(pymb_binding *binding) { + if (empty()) { + single = binding; + } else if (!is_vec()) { + vec = uintptr_t(new Vec{single, binding}) | uintptr_t(1); + } else { + as_vec()->push_back(binding); + } + } + bool erase(pymb_binding *binding) { + if (is_vec()) { + Vec *v = as_vec(); + for (auto it = v->begin(); it != v->end(); ++it) { + if (*it == binding) { + v->erase(it); + if (v->size() == 1) { + single = (*v)[0]; + delete v; + } + return true; + } + } + } else if (single == binding) { + single = nullptr; + return true; + } + return false; + } + + private: + // Discriminated pointer: points to pymb_binding if the low bit is clear, + // or to std::vector if the low bit is set. + using Vec = std::vector; + union { + pymb_binding *single; + uintptr_t vec; + }; + bool is_vec() const { return (vec & uintptr_t(1)) != 0; } + // NOLINTNEXTLINE(performance-no-int-to-ptr) + Vec *as_vec() const { return (Vec *) (vec & ~uintptr_t(1)); } + }; + + // Registered pymetabind bindings for each C++ type, including both our + // own (exported) and other frameworks' (imported). + // Protected by internals::mutex. + type_map bindings; + + // For each C++ type with a native binding, store pointers to its + // copy and move constructors. These would ideally move inside `type_info` + // on an ABI bump. Protected by internals::mutex. + type_map> copy_move_ctors; + + // List of frameworks that provide exception translators. + // Protected by internals::exception_translator_mutex. + // If this is non-empty, there is a single translator in the penultimate + // position of internals::registered_exception_translators that calls the + // translate_exception methods of each framework in this list. + std::forward_list exc_frameworks; + + // Hooks allowing other frameworks to interact with us. + // Protected by internals::mutex; constant once becoming non-null. + std::unique_ptr self; + + // Remember the C++ type associated with each binding by + // import_for_interop(), so we can clean up `bindings` properly. + // Protected by internals::mutex. + std::unordered_map manual_imports; + + // Pointer to `detail::export_for_interop` in foreign.h, or nullptr if + // export_all is false. This indirection is vital to avoid having every + // compilation unit with a py::class_ pull in the callback methods in + // foreign.h. Instead, only compilation units that call + // interoperate_by_default(), import_for_interop(), or + // export_for_interop() will emit that code. + void (*export_for_interop)(const std::type_info *, PyTypeObject *, type_info *); + + // Should we automatically advertise our types to other binding frameworks, + // or only when requested via pybind11::export_for_interop()? + // Never becomes false once it is set to true. + bool export_all = false; + + // Should we automatically use types advertised by other frameworks as + // a fallback when we can't do a cast using pybind11 types, or only when + // requested via pybind11::import_for_interop()? + // Never becomes false once it is set to true. + bool import_all = false; + + // Are there any entries in `bindings` that don't correspond to our + // own types? + bool imported_any = false; + + // Number of times the `bindings` map has been modified. Used to detect + // cases where the iterator in try_foreign_bindings() may have been + // invalidated. Protected by internals::mutex. + uint32_t bindings_update_count = 0; + + inline ~interop_internals(); + + // Returns true if we initialized, false if someone else already did. + inline bool initialize_if_needed() { + if (self) { + return false; + } + return initialize(); + } + +private: + bool initialize(); +}; + // the internals struct (above) is shared between all the modules. local_internals are only // for a single module. Any changes made to internals may require an update to // PYBIND11_INTERNALS_VERSION, breaking backwards compatibility. local_internals is, by design, @@ -348,13 +494,21 @@ struct type_info { bool module_local : 1; }; -#define PYBIND11_INTERNALS_ID \ - "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" +/// Information stored in a capsule on py::native_enum() types. +struct native_enum_info { + const std::type_info *cpptype; + uint32_t size_bytes; + bool is_signed; + + static const char *attribute_name() { return "__pybind11_enum__"; } +}; -#define PYBIND11_MODULE_LOCAL_ID \ - "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" +#define PYBIND11_ABI_TAG \ + "v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID + +#define PYBIND11_INTERNALS_ID "__pybind11_internals_" PYBIND11_ABI_TAG "__" +#define PYBIND11_MODULE_LOCAL_ID "__pybind11_module_local_v" PYBIND11_ABI_TAG "__" inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) @@ -706,6 +860,20 @@ inline auto with_exception_translators(const F &cb) local_internals.registered_exception_translators); } +inline internals_pp_manager &get_interop_internals_pp_manager() { + return internals_pp_manager::get_instance(PYBIND11_INTERNALS_ID "interop", + nullptr); +} + +inline interop_internals &get_interop_internals() { + auto &ppmgr = get_interop_internals_pp_manager(); + auto &ptr = *ppmgr.get_pp(); + if (!ptr) { + ptr.reset(new interop_internals()); + } + return *ptr; +} + inline std::uint64_t mix64(std::uint64_t z) { // David Stafford's variant 13 of the MurmurHash3 finalizer popularized // by the SplitMix PRNG. diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index a8f7675ba0..56cd121e05 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -24,15 +24,26 @@ native_enum_missing_finalize_error_message(const std::string &enum_name_encoded) class native_enum_data { public: - native_enum_data(const object &parent_scope, + native_enum_data(handle parent_scope, const char *enum_name, const char *native_type_name, const char *class_doc, - const std::type_index &enum_type_index) + const native_enum_info &enum_info_) : enum_name_encoded{enum_name}, native_type_name_encoded{native_type_name}, - enum_type_index{enum_type_index}, parent_scope(parent_scope), enum_name{enum_name}, + enum_type_index{*enum_info_.cpptype}, parent_scope(parent_scope), enum_name{enum_name}, native_type_name{native_type_name}, class_doc(class_doc), export_values_flag{false}, - finalize_needed{false} {} + finalize_needed{false} { + enum_info = capsule( + new native_enum_info{enum_info_}, + native_enum_info::attribute_name(), + +[](void *enum_info_) { + auto *info = (native_enum_info *) enum_info_; + with_internals([&](internals &internals) { + internals.native_enum_type_map.erase(*info->cpptype); + }); + delete info; + }); + } void finalize(); @@ -67,10 +78,11 @@ class native_enum_data { std::type_index enum_type_index; private: - object parent_scope; + handle parent_scope; str enum_name; str native_type_name; std::string class_doc; + capsule enum_info; protected: list members; @@ -81,12 +93,6 @@ class native_enum_data { bool finalize_needed : 1; }; -inline void global_internals_native_enum_type_map_set_item(const std::type_index &enum_type_index, - PyObject *py_enum) { - with_internals( - [&](internals &internals) { internals.native_enum_type_map[enum_type_index] = py_enum; }); -} - inline handle global_internals_native_enum_type_map_get_item(const std::type_index &enum_type_index) { return with_internals([&](internals &internals) { @@ -202,7 +208,18 @@ inline void native_enum_data::finalize() { for (auto doc : member_docs) { py_enum[doc[int_(0)]].attr("__doc__") = doc[int_(1)]; } - global_internals_native_enum_type_map_set_item(enum_type_index, py_enum.release().ptr()); + + py_enum.attr(native_enum_info::attribute_name()) = enum_info; + with_internals([&](internals &internals) { + internals.native_enum_type_map[enum_type_index] = py_enum.ptr(); + + auto &interop_internals = get_interop_internals(); + if (interop_internals.export_all) { + auto *info = enum_info.get_pointer(); + interop_internals.export_for_interop( + info->cpptype, (PyTypeObject *) py_enum.ptr(), nullptr); + } + }); } PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/detail/struct_smart_holder.h b/include/pybind11/detail/struct_smart_holder.h index 5b65b4a9b2..8830b21346 100644 --- a/include/pybind11/detail/struct_smart_holder.h +++ b/include/pybind11/detail/struct_smart_holder.h @@ -68,7 +68,7 @@ static constexpr bool type_has_shared_from_this(...) { return false; } // This overload uses SFINAE to skip enable_shared_from_this checks when the // base is inaccessible (e.g. private inheritance). template -static auto type_has_shared_from_this(const T *ptr) +static constexpr auto type_has_shared_from_this(const T *ptr) -> decltype(static_cast *>(ptr), true) { return true; } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 23d9407350..d936cecfe3 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -9,6 +9,7 @@ #pragma once +#include #include #include #include @@ -38,6 +39,11 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +// Forward declaration, implemented in foreign.h +void *try_foreign_bindings(const std::type_info *type, + void *(*attempt)(void *closure, pymb_binding *binding), + void *closure); + /// A life support system for temporary objects created by `type_caster::load()`. /// Adding a patient will keep it alive up until the enclosing function returns. class loader_life_support { @@ -99,6 +105,10 @@ class loader_life_support { Py_INCREF(h.ptr()); } } + + static bool can_add_patient() { return tls_current_frame() != nullptr; } + + const std::unordered_set &list_patients() const { return keep_alive; } }; // Gets the cache entry for the given type, creating it if necessary. The return value is the pair @@ -246,9 +256,32 @@ PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_index &tp, return nullptr; } -PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { - detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); +PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, + bool throw_if_missing, + bool foreign_ok) { + if (detail::type_info *type_info = get_type_info(tp)) { + return handle((PyObject *) type_info->type); + } + if (foreign_ok) { + auto &interop_internals = detail::get_interop_internals(); + if (interop_internals.imported_any) { + handle ret = with_internals([&](internals &) { + auto it = interop_internals.bindings.find(tp); + if (it != interop_internals.bindings.end() && !it->second.empty()) { + pymb_binding *binding = *it->second.begin(); + return handle((PyObject *) binding->pytype); + } + return handle(); + }); + if (ret) { + return ret; + } + } + } + if (throw_if_missing) { + return handle((PyObject *) get_type_info(tp, true)->type); + } + return nullptr; } inline bool try_incref(PyObject *obj) { @@ -485,8 +518,8 @@ PYBIND11_NOINLINE void instance::deallocate_layout() { } } -PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp) { - handle type = detail::get_type_handle(tp, false); +PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp, bool foreign_ok) { + handle type = detail::get_type_handle(tp, false, foreign_ok); if (!type) { return false; } @@ -777,6 +810,7 @@ struct load_helper : value_and_holder_helper { } throw std::runtime_error("Non-owning holder (load_as_shared_ptr)."); } + auto *type_raw_ptr = static_cast(void_raw_ptr); if (python_instance_is_alias && !force_potentially_slicing_shared_ptr) { auto *vptr_gd_ptr = tinfo->get_memory_guarded_delete(holder().vptr); @@ -885,21 +919,191 @@ class type_caster_generic { bool load(handle src, bool convert) { return load_impl(src, convert); } - PYBIND11_NOINLINE static handle cast(const void *_src, + // Information about how cast() can obtain its source object + struct cast_sources { + // A type-erased pointer and the type it points to + struct source { + const void *obj; + const std::type_info *type; + }; + + // Use the given pointer with its compile-time type, possibly downcast + // via polymorphic_type_hook() + template + cast_sources(const itype *ptr); // NOLINT(google-explicit-constructor) + + // Use the given pointer and type + // NOLINTNEXTLINE(google-explicit-constructor) + cast_sources(const source &orig) : original(orig) { result = resolve(); } + + // Use the given object and pybind11 type info. NB: if tinfo is null, + // this does not provide enough information to use a foreign type or + // to render a useful error message + cast_sources(const void *obj, const detail::type_info *tinfo) + : original{obj, tinfo ? tinfo->cpptype : nullptr}, result{obj, tinfo} { + if (tinfo) { + init_instance = tinfo->init_instance; + } + } + + // The object passed to cast(), with its static type. + // original.type must not be null if resolve() will be called. + // original.obj may be null if we're converting nullptr to a Python None + source original; + + // A more-derived version of `original` provided by a + // polymorphic_type_hook. downcast.type may be null if this is not + // a relevant concept for the current cast. + source downcast{}; + + // The source to use for this cast, and the corresponding pybind11 + // type_info. If the type_info is null, then pybind11 doesn't know + // about this type, but a foreign or other-module-local cast might work. + std::pair result; + + // cast() sets this to the foreign framework used, if any + mutable pymb_framework *used_foreign = nullptr; + + // Copy of type_info::init_instance, so it can be overridden in some + // cases casting a pybind11 instance on behalf of a foreign framework. + void (*init_instance)(instance *, const void *) = nullptr; + + // cast() sets this to indicate whether the cast created a new instance + // or looked up one that already existed. + mutable bool is_new = false; + + // Returns true if the cast will not succeed using a type listed in + // our pybind11 internals (it will either use a foreign-framework type + // or fail). + bool needs_foreign() const { return result.second == nullptr; } + + // Returns true if the cast will use a pybind11 type that uses + // a smart holder. + bool creates_smart_holder() const { + return result.second != nullptr + && result.second->holder_enum_v == detail::holder_enum_t::smart_holder; + } + + private: + std::pair resolve() { + if (downcast.type) { + if (same_type(*original.type, *downcast.type)) { + downcast.type = nullptr; + } else if (const auto *tpi = get_type_info(*downcast.type)) { + init_instance = tpi->init_instance; + return {downcast.obj, tpi}; + } + } + if (const auto *tpi = get_type_info(*original.type)) { + init_instance = tpi->init_instance; + return {original.obj, tpi}; + } + return {nullptr, nullptr}; + } + }; + + static handle cast(const void *src, + return_value_policy policy, + handle parent, + const detail::type_info *tinfo, + copy_or_move_ctor copy_constructor, + copy_or_move_ctor move_constructor, + const void *existing_holder = nullptr) { + cast_sources srcs{src, tinfo}; + return cast(srcs, policy, parent, copy_constructor, move_constructor, existing_holder); + } + + static handle cast_foreign(const cast_sources &srcs, + return_value_policy policy, + handle parent, + bool has_holder) { + struct capture { + const void *src; + pymb_rv_policy policy = pymb_rv_policy_none; + pymb_to_python_feedback feedback{}; + const cast_sources *srcs; + } cap; + + auto attempt = +[](void *closure, pymb_binding *binding) -> void * { + capture &cap = *(capture *) closure; + void *ret = binding->framework->to_python( + binding, const_cast(cap.src), cap.policy, &cap.feedback); + if (ret) { + cap.srcs->used_foreign = binding->framework; + cap.srcs->is_new = cap.feedback.is_new; + } + return ret; + }; + + cap.srcs = &srcs; + switch (policy) { + case return_value_policy::automatic: + cap.policy = pymb_rv_policy_take_ownership; + break; + case return_value_policy::automatic_reference: + case return_value_policy::reference: + cap.policy + = has_holder ? pymb_rv_policy_share_ownership : pymb_rv_policy_reference; + break; + case return_value_policy::reference_internal: + if (!parent) { + return nullptr; + } + cap.policy = pymb_rv_policy_share_ownership; + break; + case return_value_policy::take_ownership: + case return_value_policy::copy: + case return_value_policy::move: + cap.policy = (pymb_rv_policy) (uint8_t) policy; + break; + } + + void *result_v = nullptr; + if (srcs.downcast.type) { + cap.src = srcs.downcast.obj; + result_v = try_foreign_bindings(srcs.downcast.type, attempt, &cap); + } + if (!result_v && srcs.original.type) { + cap.src = srcs.original.obj; + result_v = try_foreign_bindings(srcs.original.type, attempt, &cap); + } + + auto *result = (PyObject *) result_v; + if (result && policy == return_value_policy::reference_internal && srcs.is_new + && srcs.used_foreign->keep_alive(result, parent.ptr(), nullptr) == 0) { + keep_alive_impl(result, parent.ptr()); + } + return result; + } + + PYBIND11_NOINLINE static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent, - const detail::type_info *tinfo, - void *(*copy_constructor)(const void *), - void *(*move_constructor)(const void *), + copy_or_move_ctor copy_constructor, + copy_or_move_ctor move_constructor, const void *existing_holder = nullptr) { - if (!tinfo) { // no type info: error will be set already + if (!srcs.result.second) { + // No pybind11 type info. See if we can use another framework's + // type to complete this cast. Set srcs.used_foreign if so. + if (srcs.original.type && get_interop_internals().imported_any) { + if (handle ret = cast_foreign(srcs, policy, parent, existing_holder != nullptr)) { + return ret; + } + } + std::string tname = srcs.downcast.type ? srcs.downcast.type->name() + : srcs.original.type ? srcs.original.type->name() + : ""; + detail::clean_type_id(tname); + std::string msg = "Unregistered type : " + tname; + set_error(PyExc_TypeError, msg.c_str()); return handle(); } - void *src = const_cast(_src); + void *src = const_cast(srcs.result.first); if (src == nullptr) { return none().release(); } + const type_info *tinfo = srcs.result.second; if (handle registered_inst = find_registered_python_instance(src, tinfo)) { return registered_inst; @@ -972,7 +1176,8 @@ class type_caster_generic { throw cast_error("unhandled return_value_policy: should not happen!"); } - tinfo->init_instance(wrapper, existing_holder); + srcs.init_instance(wrapper, existing_holder); + srcs.is_new = true; return inst.release(); } @@ -1034,6 +1239,7 @@ class type_caster_generic { return false; } void check_holder_compat() {} + bool set_foreign_holder(handle) { return true; } PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { auto caster = type_caster_generic(ti); @@ -1043,43 +1249,78 @@ class type_caster_generic { return nullptr; } - /// Try to load with foreign typeinfo, if available. Used when there is no - /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { - constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; - const auto pytype = type::handle_of(src); - if (!hasattr(pytype, local_key)) { + /// Try to load as a type exposed by a different binding framework (which + /// might be an ABI-incompatible version of pybind11). + bool try_load_other_framework(handle src, bool convert) { + auto &interop_internals = get_interop_internals(); + if (!interop_internals.imported_any || !cpptype || src.is_none()) { return false; } - type_info *foreign_typeinfo = reinterpret_borrow(getattr(pytype, local_key)); - // Only consider this foreign loader if actually foreign and is a loader of the correct cpp - // type - if (foreign_typeinfo->module_local_load == &local_load - || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) { + struct capture { + handle src; + bool convert; + } cap{src, convert}; + + auto attempt = +[](void *closure, pymb_binding *binding) -> void * { + capture &cap = *(capture *) closure; + void (*keep_referenced)(void *, PyObject *) = nullptr; + if (loader_life_support::can_add_patient()) { + keep_referenced + = [](void *, PyObject *item) { loader_life_support::add_patient(item); }; + } + return binding->framework->from_python( + binding, cap.src.ptr(), cap.convert, keep_referenced, nullptr); + }; + + if (void *result = try_foreign_bindings(cpptype, attempt, &cap)) { + value = result; + return true; + } + return false; + } + + /// Try to load as a type bound as py::module_local() in a different (but + /// ABI-compatible) pybind11 module. + bool try_load_other_module_local(handle src, type_info *remote_typeinfo) { + // Only consider this loader if it's not ours and it loads the correct cpp type + if (remote_typeinfo->module_local_load == &local_load + || (cpptype && !same_type(*cpptype, *remote_typeinfo->cpptype))) { return false; } - if (auto *result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { + if (auto *result = remote_typeinfo->module_local_load(src.ptr(), remote_typeinfo)) { value = result; return true; } return false; } + /// Try to load with foreign typeinfo, if available. Used when there is no + /// native typeinfo, or when the native one wasn't able to produce a value. + PYBIND11_NOINLINE bool try_load_foreign(handle src, bool convert, bool foreign_ok) { + constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; + const auto pytype = type::handle_of(src); + if (hasattr(pytype, local_key)) { + return try_load_other_module_local( + src, reinterpret_borrow(getattr(pytype, local_key))); + } + return foreign_ok && try_load_other_framework(src, convert); + } + // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant // bits of code between here and copyable_holder_caster where the two classes need different // logic (without having to resort to virtual inheritance). template - PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { + PYBIND11_NOINLINE bool load_impl(handle src, bool convert, bool foreign_ok = true) { + auto &this_ = static_cast(*this); if (!src) { return false; } if (!typeinfo) { - return try_load_foreign_module_local(src); + return try_load_foreign(src, convert, foreign_ok) && this_.set_foreign_holder(src); } - auto &this_ = static_cast(*this); this_.check_holder_compat(); PyTypeObject *srctype = Py_TYPE(src.ptr()); @@ -1145,13 +1386,14 @@ class type_caster_generic { if (typeinfo->module_local) { if (auto *gtype = get_global_type_info(*typeinfo->cpptype)) { typeinfo = gtype; - return load(src, false); + return load_impl(src, false); } } - // Global typeinfo has precedence over foreign module_local - if (try_load_foreign_module_local(src)) { - return true; + // Global typeinfo has precedence over foreign module_local and + // foreign frameworks + if (try_load_foreign(src, convert, foreign_ok)) { + return this_.set_foreign_holder(src); } // Custom converters didn't take None, now we convert None to nullptr. @@ -1165,31 +1407,12 @@ class type_caster_generic { } if (convert && cpptype && this_.try_cpp_conduit(src)) { - return true; + return this_.set_foreign_holder(src); } return false; } - // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast - // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair - // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). - PYBIND11_NOINLINE static std::pair - src_and_type(const void *src, - const std::type_info &cast_type, - const std::type_info *rtti_type = nullptr) { - if (auto *tpi = get_type_info(cast_type)) { - return {src, const_cast(tpi)}; - } - - // Not found, set error: - std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); - detail::clean_type_id(tname); - std::string msg = "Unregistered type : " + tname; - set_error(PyExc_TypeError, msg.c_str()); - return {nullptr, nullptr}; - } - const type_info *typeinfo = nullptr; const std::type_info *cpptype = nullptr; void *value = nullptr; @@ -1484,6 +1707,20 @@ struct polymorphic_type_hook : public polymorphic_type_hook_base {}; PYBIND11_NAMESPACE_BEGIN(detail) +template +type_caster_generic::cast_sources::cast_sources(const itype *ptr) : original{ptr, &typeid(itype)} { + // If this is a base pointer to a derived type, and the derived type is + // registered with pybind11, we want to make the full derived object + // available. In the typical case where itype is polymorphic, we get the + // correct derived pointer (which may be != base pointer) by a dynamic_cast + // to most derived type. If itype is not polymorphic, a user-provided + // specialization of polymorphic_type_hook can do the same thing. + // If there is no downcast to perform, then the default hook will leave + // derived.type set to nullptr, which causes us to ignore derived.obj. + downcast.obj = polymorphic_type_hook::get(ptr, downcast.type); + result = resolve(); +} + /// Generic type caster for objects stored on the heap template class type_caster_base : public type_caster_generic { @@ -1495,6 +1732,11 @@ class type_caster_base : public type_caster_generic { type_caster_base() : type_caster_base(typeid(type)) {} explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) {} + struct cast_sources : type_caster_generic::cast_sources { + // NOLINTNEXTLINE(google-explicit-constructor) + cast_sources(const itype *ptr) : type_caster_generic::cast_sources(ptr) {} + }; + static handle cast(const itype &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { @@ -1507,50 +1749,83 @@ class type_caster_base : public type_caster_generic { return cast(std::addressof(src), return_value_policy::move, parent); } - // Returns a (pointer, type_info) pair taking care of necessary type lookup for a - // polymorphic type (using RTTI by default, but can be overridden by specializing - // polymorphic_type_hook). If the instance isn't derived, returns the base version. - static std::pair src_and_type(const itype *src) { - const auto &cast_type = typeid(itype); - const std::type_info *instance_type = nullptr; - const void *vsrc = polymorphic_type_hook::get(src, instance_type); - if (instance_type && !same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type. If the derived type is registered - // with pybind11, we want to make the full derived object available. - // In the typical case where itype is polymorphic, we get the correct - // derived pointer (which may be != base pointer) by a dynamic_cast to - // most derived type. If itype is not polymorphic, we won't get here - // except via a user-provided specialization of polymorphic_type_hook, - // and the user has promised that no this-pointer adjustment is - // required in that case, so it's OK to use static_cast. - if (const auto *tpi = get_type_info(*instance_type)) { - return {vsrc, tpi}; - } - } - // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, - // so don't do a cast - return type_caster_generic::src_and_type(src, cast_type, instance_type); - } - - static handle cast(const itype *src, return_value_policy policy, handle parent) { - auto st = src_and_type(src); - return type_caster_generic::cast(st.first, + static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent) { + return type_caster_generic::cast(srcs, policy, parent, - st.second, - make_copy_constructor(src), - make_move_constructor(src)); - } - - static handle cast_holder(const itype *src, const void *holder) { - auto st = src_and_type(src); - return type_caster_generic::cast(st.first, - return_value_policy::take_ownership, - {}, - st.second, - nullptr, - nullptr, - holder); + make_copy_constructor((const itype *) nullptr), + make_move_constructor((const itype *) nullptr)); + } + + PYBIND11_NOINLINE static void after_shared_ptr_cast_to_foreign( + handle ret, std::shared_ptr holder, pymb_framework *framework) { + // Make the resulting Python object keep a shared_ptr alive, + // even if there's not space for it inside the object. + std::unique_ptr> sp{new auto(std::move(holder))}; + auto deleter = [](void *p) noexcept { delete (std::shared_ptr *) p; }; + if (!framework->keep_alive(ret.ptr(), sp.get(), deleter)) { + // If they don't provide a keep_alive, use our own weakref-based + // one. If ret is not weakrefable, it will throw and the capsule's + // destructor will clean up for us. + keep_alive_impl(ret, capsule((void *) sp.release(), deleter)); + } + (void) sp.release(); // NOLINT(bugprone-unused-return-value) + } + + template + static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { + handle ret = type_caster_generic::cast( + srcs, return_value_policy::take_ownership, {}, nullptr, nullptr, holder); + if (srcs.used_foreign) { + // Foreign cast succeeded; transfer ownership + if (srcs.is_new) { + // The new instance took ownership, so drop it on the C++ side + (void) holder->release(); // NOLINT(bugprone-unused-return-value) + } else { + // The instance already existed, so wouldn't have been carrying + // ownership since `*holder` held exclusive ownership until a + // moment ago. Transfer the ownership via a keep_alive instead. + after_shared_ptr_cast_to_foreign( + ret, std::shared_ptr{std::move(*holder)}, srcs.used_foreign); + } + } + return ret; + } + + template + static handle cast_holder(const cast_sources &srcs, const std::shared_ptr *holder) { + // Use reference policy if casting via a foreign binding, and + // take_ownership if casting a pybind11 type + auto policy = srcs.needs_foreign() ? return_value_policy::reference + : return_value_policy::take_ownership; + handle ret = type_caster_generic::cast(srcs, policy, {}, nullptr, nullptr, holder); + if (srcs.used_foreign && srcs.is_new) { + after_shared_ptr_cast_to_foreign( + ret, std::static_pointer_cast(*holder), srcs.used_foreign); + } + return ret; + } + + // Support unique_ptr with custom deleter by converting it to shared_ptr + template + static handle cast_holder(const cast_sources &srcs, std::unique_ptr *holder) { + std::shared_ptr shared_holder = std::move(*holder); + return cast_holder(srcs, &shared_holder); + } + + static handle cast_holder(const cast_sources &srcs, const void *holder) { + auto policy = srcs.needs_foreign() ? return_value_policy::reference + : return_value_policy::take_ownership; + handle ret = type_caster_generic::cast(srcs, policy, {}, nullptr, nullptr, holder); + if (srcs.used_foreign) { + PyErr_SetString(PyExc_TypeError, + "Can't cast foreign type to holder; only " + "returns of std::unique_ptr and std::shared_ptr " + "are supported for foreign types"); + ret.dec_ref(); + return nullptr; + } + return ret; } template @@ -1567,27 +1842,31 @@ class type_caster_base : public type_caster_generic { } protected: - using Constructor = void *(*) (const void *); - /* Only enabled when the types are {copy,move}-constructible *and* when the type does not have a private operator new implementation. A comma operator is used in the decltype argument to apply SFINAE to the public copy/move constructors.*/ template ::value>> static auto make_copy_constructor(const T *) - -> decltype(new T(std::declval()), Constructor{}) { + -> decltype(new T(std::declval()), copy_or_move_ctor{}) { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } template ::value>> static auto make_move_constructor(const T *) - -> decltype(new T(std::declval()), Constructor{}) { + -> decltype(new T(std::declval()), copy_or_move_ctor{}) { return [](const void *arg) -> void * { return new T(std::move(*const_cast(reinterpret_cast(arg)))); }; } - static Constructor make_copy_constructor(...) { return nullptr; } - static Constructor make_move_constructor(...) { return nullptr; } + static copy_or_move_ctor make_copy_constructor(...) { return nullptr; } + static copy_or_move_ctor make_move_constructor(...) { return nullptr; } + +public: + static std::pair copy_and_move_ctors() { + return {make_copy_constructor(static_cast(nullptr)), + make_move_constructor(static_cast(nullptr))}; + } }; inline std::string quote_cpp_type_name(const std::string &cpp_type_name) { @@ -1595,8 +1874,7 @@ inline std::string quote_cpp_type_name(const std::string &cpp_type_name) { } PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { - if (auto *type_data = get_type_info(ti)) { - handle th((PyObject *) type_data->type); + if (handle th = get_type_handle(ti, false, true)) { return th.attr("__module__").cast() + '.' + th.attr("__qualname__").cast(); } diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index a820bfbfc3..47d7e33aa2 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -245,6 +245,7 @@ inline void finalize_interpreter() { if (detail::get_num_interpreters_seen() > 1) { detail::get_internals_pp_manager().unref(); detail::get_local_internals_pp_manager().unref(); + detail::get_interop_internals_pp_manager().unref(); // We know there can be no other interpreter alive now, so we can lower the count detail::get_num_interpreters_seen() = 1; @@ -256,14 +257,16 @@ inline void finalize_interpreter() { // and check it after Py_Finalize(). detail::get_internals_pp_manager().get_pp(); detail::get_local_internals_pp_manager().get_pp(); + detail::get_interop_internals_pp_manager().get_pp(); Py_Finalize(); + // Internals contain data managed by the current interpreter, so we must + // clear them to avoid undefined behaviors when initializing another + // interpreter detail::get_internals_pp_manager().destroy(); - - // Local internals contains data managed by the current interpreter, so we must clear them to - // avoid undefined behaviors when initializing another interpreter detail::get_local_internals_pp_manager().destroy(); + detail::get_interop_internals_pp_manager().destroy(); // We know there is no interpreter alive now, so we can reset the count detail::get_num_interpreters_seen() = 0; diff --git a/include/pybind11/native_enum.h b/include/pybind11/native_enum.h index 5537218f21..72e124c7a8 100644 --- a/include/pybind11/native_enum.h +++ b/include/pybind11/native_enum.h @@ -22,12 +22,11 @@ class native_enum : public detail::native_enum_data { public: using Underlying = typename std::underlying_type::type; - native_enum(const object &parent_scope, + native_enum(handle parent_scope, const char *name, const char *native_type_name, const char *class_doc = "") - : detail::native_enum_data( - parent_scope, name, native_type_name, class_doc, std::type_index(typeid(EnumType))) { + : detail::native_enum_data(parent_scope, name, native_type_name, class_doc, make_info()) { if (detail::get_local_type_info(typeid(EnumType)) != nullptr || detail::get_global_type_info(typeid(EnumType)) != nullptr) { pybind11_fail( @@ -62,6 +61,15 @@ class native_enum : public detail::native_enum_data { native_enum(const native_enum &) = delete; native_enum &operator=(const native_enum &) = delete; + +private: + static detail::native_enum_info make_info() { + detail::native_enum_info ret; + ret.cpptype = &typeid(EnumType); + ret.size_bytes = sizeof(EnumType); + ret.is_signed = std::is_signed::value; + return ret; + } }; PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 117fecabf0..ce51188057 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -12,6 +12,7 @@ #include "detail/class.h" #include "detail/dynamic_raw_ptr_cast_if_possible.h" #include "detail/exception_translation.h" +#include "detail/foreign.h" #include "detail/function_record_pyobject.h" #include "detail/init.h" #include "detail/native_enum_data.h" @@ -157,11 +158,11 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel if (!t) { pybind11_fail("Internal error while parsing type signature (1)"); } - if (auto *tinfo = detail::get_type_info(*t)) { - handle th((PyObject *) tinfo->type); - signature += th.attr("__module__").cast() + "." - + th.attr("__qualname__").cast(); - } else if (auto th = detail::global_internals_native_enum_type_map_get_item(*t)) { + handle th = detail::get_type_handle(*t, false, true); + if (!th) { + th = detail::global_internals_native_enum_type_map_get_item(*t); + } + if (th) { signature += th.attr("__module__").cast() + "." + th.attr("__qualname__").cast(); } else if (func_rec->is_new_style_constructor && arg_index == 0) { @@ -242,12 +243,6 @@ inline std::string generate_type_signature() { caster_name_field.text, &func_rec, descr_types.data(), type_index, arg_index); } -#if defined(_MSC_VER) -# define PYBIND11_COMPAT_STRDUP _strdup -#else -# define PYBIND11_COMPAT_STRDUP strdup -#endif - PYBIND11_NAMESPACE_END(detail) /// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object @@ -1652,6 +1647,11 @@ class generic_type : public object { #endif internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; PYBIND11_WARNING_POP + + auto &interop_internals = get_interop_internals(); + if (interop_internals.export_all) { + interop_internals.export_for_interop(rec.type, (PyTypeObject *) m_ptr, tinfo); + } }); if (rec.bases.size() > 1 || rec.multiple_inheritance) { @@ -2135,14 +2135,16 @@ class class_ : public detail::generic_type { generic_type::initialize(record); - if (has_alias) { - with_internals([&](internals &internals) { + with_internals([&](internals &internals) { + get_interop_internals().copy_move_ctors.emplace( + *record.type, detail::type_caster_base::copy_and_move_ctors()); + if (has_alias) { auto &instances = record.module_local ? get_local_internals().registered_types_cpp : internals.registered_types_cpp; instances[std::type_index(typeid(type_alias))] = instances[std::type_index(typeid(type))]; - }); - } + } + }); def("_pybind11_conduit_v1_", cpp_conduit_method); } @@ -2924,12 +2926,19 @@ PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { return; /* Nothing to keep alive or nothing to be kept alive by */ } - auto tinfo = all_type_info(Py_TYPE(nurse.ptr())); + PyTypeObject *type = Py_TYPE(nurse.ptr()); + auto tinfo = all_type_info(type); if (!tinfo.empty()) { /* It's a pybind-registered type, so we can store the patient in the * internal list. */ add_patient(nurse.ptr(), patient.ptr()); } else { + auto *binding = pymb_get_binding((PyObject *) type); + if (binding && binding->framework->keep_alive(nurse.ptr(), patient.ptr(), nullptr) != 0) { + // It's a foreign-registered type and the foreign framework was + // able to handle the keep_alive. + return; + } /* Fall back to clever approach based on weak references taken from * Boost.Python. This is not used for pybind-registered types because * the objects can be destroyed out-of-order in a GC pass. */ diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index cee4ab5623..e0815cd32a 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -46,7 +46,7 @@ struct arg_v; PYBIND11_NAMESPACE_BEGIN(detail) class args_proxy; -bool isinstance_generic(handle obj, const std::type_info &tp); +bool isinstance_generic(handle obj, const std::type_info &tp, bool foreign_ok); template bool isinstance_native_enum(handle obj, const std::type_info &tp); @@ -855,13 +855,13 @@ bool isinstance(handle obj) { } template ::value, int> = 0> -bool isinstance(handle obj) { +bool isinstance(handle obj, bool foreign_ok = false) { return detail::isinstance_native_enum(obj, typeid(T)) - || detail::isinstance_generic(obj, typeid(T)); + || detail::isinstance_generic(obj, typeid(T), foreign_ok); } template <> -inline bool isinstance(handle) = delete; +inline bool isinstance(handle, bool) = delete; template <> inline bool isinstance(handle obj) { return obj.ptr() != nullptr; @@ -1573,14 +1573,14 @@ class type : public object { /// standard types, like int, float. etc. yet. /// See https://github.com/pybind/pybind11/issues/2486 template - static handle handle_of(); + static handle handle_of(bool foreign_ok = false); /// Convert C++ type to type if previously registered. Does not convert /// standard types, like int, float. etc. yet. /// See https://github.com/pybind/pybind11/issues/2486 template - static type of() { - return type(type::handle_of(), borrowed_t{}); + static type of(bool foreign_ok = false) { + return type(type::handle_of(foreign_ok), borrowed_t{}); } }; diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 4b208ed9b6..de4cd4a5ee 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -180,6 +180,7 @@ class subinterpreter { // internals themselves. detail::get_internals_pp_manager().get_pp(); detail::get_local_internals_pp_manager().get_pp(); + detail::get_interop_internals_pp_manager().get_pp(); // End it Py_EndInterpreter(destroy_tstate); @@ -188,6 +189,7 @@ class subinterpreter { // py::capsule calls `get_internals()` during destruction), so we destroy afterward. detail::get_internals_pp_manager().destroy(); detail::get_local_internals_pp_manager().destroy(); + detail::get_interop_internals_pp_manager().destroy(); // switch back to the old tstate and old GIL (if there was one) if (switch_back) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 34b22a57ae..801aabc994 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -150,6 +150,7 @@ set(PYBIND11_TEST_FILES test_exceptions test_factory_constructors test_gil_scoped + test_interop.py test_iostream test_kwargs_and_defaults test_local_bindings @@ -247,6 +248,7 @@ tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils") tests_extra_targets("test_cpp_conduit.py" "exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler") +tests_extra_targets("test_interop.py" "test_interop_1;test_interop_2;test_interop_3") set(PYBIND11_EIGEN_REPO "https://gitlab.com/libeigen/eigen.git" diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 63e59f65a6..ef4eac4655 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -81,6 +81,7 @@ "include/pybind11/detail/cpp_conduit.h", "include/pybind11/detail/descr.h", "include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h", + "include/pybind11/detail/foreign.h", "include/pybind11/detail/function_record_pyobject.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", @@ -94,6 +95,10 @@ "include/pybind11/detail/exception_translation.h", } +contrib_headers = { + "include/pybind11/contrib/pymetabind.h", +} + eigen_headers = { "include/pybind11/eigen/common.h", "include/pybind11/eigen/matrix.h", @@ -130,7 +135,14 @@ "share/pkgconfig/__init__.py", } -headers = main_headers | conduit_headers | detail_headers | eigen_headers | stl_headers +headers = ( + main_headers + | conduit_headers + | detail_headers + | contrib_headers + | eigen_headers + | stl_headers +) generated_files = cmake_files | pkgconfig_files all_files = headers | generated_files | py_files diff --git a/tests/test_interop.h b/tests/test_interop.h new file mode 100644 index 0000000000..42af357f5c --- /dev/null +++ b/tests/test_interop.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include + +#include + +struct PYBIND11_EXPORT_EXCEPTION SharedExc { + int value; +}; + +struct Shared { + int value; + + enum class Enum { One = 1, Two = 2 }; + + struct Stats { + std::atomic constructed{0}; + std::atomic copied{0}; + std::atomic moved{0}; + std::atomic destroyed{0}; + }; + static Stats &stats() { + static Stats st; + return st; + } + + // NOLINTNEXTLINE(google-explicit-constructor) + Shared(int v = 0) : value(v) { ++stats().constructed; } + Shared(const Shared &other) : value(other.value) { ++stats().copied; } + Shared(Shared &&other) noexcept : value(other.value) { ++stats().moved; } + ~Shared() { ++stats().destroyed; } + + static Shared make(int v) { return Shared{v}; } + static std::shared_ptr make_sp(int v) { return std::make_shared(v); } + static std::unique_ptr make_up(int v) { + return std::unique_ptr(new Shared{v}); + } + static Enum make_enum(int v) { return Enum(v); } + + static int check(const Shared &s) { return s.value; } + // NOLINTNEXTLINE(performance-unnecessary-value-param) + static int check_sp(std::shared_ptr s) { return s->value; } + static int check_up(std::unique_ptr s) { return s->value; } + static int check_enum(Enum e) { return (int) e; } + + static long uses(const std::shared_ptr &s) { return s.use_count(); } + + static pybind11::dict pull_stats() { + pybind11::dict ret; + auto &st = stats(); + ret["construct"] = st.constructed.exchange(0); + ret["copy"] = st.copied.exchange(0); + ret["move"] = st.moved.exchange(0); + ret["destroy"] = st.destroyed.exchange(0); + return ret; + } + + template + static void bind_funcs(pybind11::module_ m) { + m.def("make", &make); + m.def("make_sp", &make_sp); + if (SmartHolder) { + m.def("make_up", &make_up); + } else { + // non-smart holder can't bind a unique_ptr return when the + // holder type is shared_ptr + m.def( + "make_up", + [](int v) { return make_up(v).release(); }, + pybind11::return_value_policy::take_ownership); + } + m.def("make_enum", &make_enum); + m.def("check", &check); + m.def("check_sp", &check_sp); + m.def("check_up", &check_up); + m.def("check_enum", &check_enum); + m.def("uses", &uses); + m.def("pull_stats", &pull_stats); + + m.def("export_all", []() { pybind11::interoperate_by_default(true, false); }); + m.def("import_all", []() { pybind11::interoperate_by_default(false, true); }); + m.def("export_for_interop", &pybind11::export_for_interop); + m.def("import_for_interop", &pybind11::import_for_interop<>); + m.def("import_for_interop_explicit", &pybind11::import_for_interop); + struct Other {}; + m.def("import_for_interop_wrong_type", &pybind11::import_for_interop); + } + + template + static void bind_types(pybind11::handle scope) { + if (pybind11::hasattr(scope, "Shared")) { + if (pybind11::detail::get_interop_internals().export_all) { + // If bindings were removed but types weren't (because types + // are immortal in this environment) then manually re-export + // the bindings so that the effects of export_all are observable + pybind11::export_for_interop(scope.attr("Shared")); + pybind11::export_for_interop(scope.attr("SharedEnum")); + } + return; + } + using Holder = typename std:: + conditional>::type; + pybind11::class_(scope, "Shared").def_readonly("value", &Shared::value); + pybind11::native_enum(scope, "SharedEnum", "enum.Enum") + .value("One", Enum::One) + .value("Two", Enum::Two) + .finalize(); + pybind11::delattr(scope.attr("Shared"), "_pybind11_conduit_v1_"); + } +}; diff --git a/tests/test_interop.py b/tests/test_interop.py new file mode 100644 index 0000000000..477880d413 --- /dev/null +++ b/tests/test_interop.py @@ -0,0 +1,588 @@ +# Copyright (c) 2025 The pybind from __future__ import annotations +from __future__ import annotations + +import collections +import gc +import itertools +import sys +import sysconfig +import threading +import time +import weakref + +import pytest +import test_interop_1 as t1 +import test_interop_2 as t2 +import test_interop_3 as t3 + +import env + +free_threaded = hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() +types_are_immortal = sys.implementation.name in ("graalpy", "pypy") or ( + sysconfig.get_config_var("Py_GIL_DISABLED") and sys.version_info < (3, 14) +) + +# t1, t2, t3 all define bindings for the same C++ type `Shared`, as well as +# several functions to create and inspect instances of that type. Each module +# uses a different PYBIND11_INTERNALS_VERSION, so they won't interoperate +# natively; these tests check the foreign-framework interoperability mechanism. +# The bindings are all different, as follows: +# t1.Shared uses a std::shared_ptr holder +# t2.Shared uses a smart_holder +# t3.RawShared uses the Python C API to mimic a non-pybind11 framework +# Upon import, bindings are defined for the functions, but not for the +# types (`Shared` and `SharedEnum`) until you call bind_types(). +# +# NB: there is some potential for different test cases to interfere with +# each other: we can't un-register a framework once it's registered and we +# can't undo automatic import/export all once they're requested. The ordering +# of these tests is therefore important. They work standalone and they work +# when run all in one process, but they might not work in a different order. + + +def delattr_and_ensure_destroyed(*specs): + wrs = [] + for mod, name in specs: + wrs.append(weakref.ref(getattr(mod, name))) + delattr(mod, name) + + for _ in range(5): + gc.collect() + if all(wr() is None for wr in wrs): + break + else: + pytest.fail( + f"Could not delete bindings such as {next(wr for wr in wrs if wr() is not None)!r}" + ) + + +@pytest.fixture(autouse=True) +def clean_after(): + yield + if sys.implementation.name in ("pypy", "graalpy"): + pytest.gc_collect() + if sys.implementation.name != "graalpy": + t3.clear_interop_bindings() + + if not types_are_immortal: + delattr_and_ensure_destroyed( + *[ + (mod, name) + for mod in (t1, t2, t3) + for name in ("Shared", "SharedEnum", "RawShared") + if hasattr(mod, name) + ] + ) + + t1.pull_stats() + t2.pull_stats() + t3.pull_stats() + + +def check_stats(mod, **entries): + if mod is None or sys.implementation.name == "graalpy": + # graalpy seems to not do a full collection when we gc_collect(); there + # will be too few destructions in one check_stats() and then correspondingly + # too many in the next one for the same module, so just skip the check + return + if sys.implementation.name == "pypy": + pytest.gc_collect() + stats = mod.pull_stats() + if stats["move"] == entries.get("move", 0) + 1: + # Allow an extra move+destroy pair to account for older compilers + # not doing RVO like we expect. + entries["move"] = entries.get("move", 0) + 1 + entries["destroy"] = entries.get("destroy", 0) + 1 + for name, value in entries.items(): + assert stats.pop(name) == value + assert all(val == 0 for val in stats.values()) + + +def test01_interop_exceptions_without_registration(): + # t2 defines the exception translator for Shared. Since it hasn't + # taken any interop actions yet, it hasn't registered with pymetabind + # and t1 won't be able to use that translator. + with pytest.raises(RuntimeError, match="Caught an unknown exception"): + t1.throw_shared(100) + + with pytest.raises(ValueError, match="Shared.200"): + t2.throw_shared(200) + + +global_counter = itertools.count() + + +def expect(from_mod, to_mod, pattern, **extra): + outcomes = {} + extra_info = {} + owner_mod = None + + for idx, suffix in enumerate(("", "_sp", "_up", "_enum")): + create = getattr(from_mod, f"make{suffix}") + check = getattr(to_mod, f"check{suffix}") + thing = suffix.lstrip("_") or "value" + print(thing) + value = idx * 1000 + next(global_counter) + if thing == "enum": + value = (value % 2) + 1 + try: + obj = create(value) + except Exception as ex: + outcomes[thing] = None + extra_info[thing] = ex + continue + if owner_mod is None: + owner_mod = sys.modules[type(obj).__module__] + try: + roundtripped = check(obj) + except Exception as ex: + outcomes[thing] = False + extra_info[thing] = ex + continue + assert roundtripped == value, "instance appears corrupted" + if thing == "sp": + # Include shared_ptr use count in the test. Foreign should create + # a new control block so we see use_count == 1. Local should reuse + # the same -> use_count == 0. + outcomes[thing] = to_mod.uses(obj) + else: + outcomes[thing] = True + + expected = {} + if pattern == "local": + # Accepting a unique_ptr argument only works for non-foreign smart_holder + expected = {"value": True, "sp": 2, "up": to_mod is t2, "enum": True} + elif pattern == "foreign": + expected = {"value": True, "sp": 1, "up": False, "enum": True} + elif pattern == "isolated": + expected = {"value": False, "sp": False, "up": False, "enum": False} + elif pattern == "none": + expected = {"value": None, "sp": None, "up": None, "enum": None} + else: + pytest.fail("unknown pattern") + expected.update(extra) + assert outcomes == expected + + obj = None + + # When returning by value, we have a construction in from_mod, + # move to owner_mod, destruction in from_mod (after make() returns) + # and destruction in owner mod (when the pyobject dies). + # + # When returning shared_ptr, the construction and destruction both + # occur in from_mod since shared_ptr's deleter is set at creation time. + # + # When returning unique_ptr, the construction occurs in from_mod and + # destruction (when the pyobject dies) occurs in owner_mod; unless + # we pass ownership to to_mod, in which case the destruction happens there. + # But since we can't pass ownership to a foreign framework currently, + # we'll disregard that possibility and always attribute it to owner_mod. + expect_stats = {mod: collections.Counter() for mod in (from_mod, to_mod, owner_mod)} + expect_stats[from_mod].update( + ["construct", "destroy", "construct", "destroy", "construct"] + ) + # value move+destroy + expect_stats[owner_mod].update(["move", "destroy"]) + # unique_ptr destroy; due to an existing pybind11 bug this may be skipped + # entirely (leaked) if we return a raw pointer with rvp take_ownership and + # the cast fails + if owner_mod is None and from_mod is t1: + pass + else: + expect_stats[owner_mod or from_mod].update(["destroy"]) + for mod, stats in expect_stats.items(): + check_stats(mod, **stats) + + +def test02a_interop_return_foreign_smart_holder(): + # Test a pybind11 domain returning a different pybind11 domain's type + # because it didn't have its own. + t2.bind_types() + t2.export_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t1.import_for_interop(t2.Shared) + t1.import_for_interop(t2.SharedEnum) + expect(t2, t1, "foreign") + expect(t1, t2, "local") # t2 is both source and dest + assert type(t1.make(1)) is t2.Shared + assert type(t2.make(2)) is t2.Shared + + +def test02b_interop_return_foreign_shared_ptr(): + # Same thing but different holder type (t2 uses smart holder, t1 regular). + t1.bind_types() + t1.export_for_interop(t1.Shared) + t1.export_for_interop(t1.SharedEnum) + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t1.SharedEnum) + expect(t1, t3, "foreign") + expect(t3, t1, "local") # t1 is both source and dest + assert type(t1.make(1)) is t1.Shared + assert type(t3.make(2)) is t1.Shared + + +def test03_interop_unimported(): + # Bind the types but don't share them yet + t1.bind_types() + t2.bind_types() + + for mod in (t1, t2): + expect(mod, mod, "local") + + # t3 hasn't defined SharedEnum yet. Its version of Shared is not + # bound using pybind11, so is foreign even to the functions in t3. + t3.create_raw_binding() + expect(t3, t3, "foreign", enum=None) + + expect(t1, t2, "isolated") + expect(t1, t3, "isolated") + expect(t2, t1, "isolated") + expect(t2, t3, "isolated") + expect(t3, t1, "isolated", enum=None) + expect(t3, t2, "isolated", enum=None) + + # Just an export isn't enough; you need an import too + t2.export_for_interop(t2.Shared) + expect(t2, t3, "isolated") + + +def test04_interop_import_export_errors(): + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + + with pytest.raises( + RuntimeError, match="type does not define a __pymetabind_binding__" + ): + t2.import_for_interop(t1.Convertible) + + with pytest.raises(RuntimeError, match="not a pybind11 class or enum"): + t3.export_for_interop(t2.Shared) + + with pytest.raises(RuntimeError, match="not a pybind11 class or enum"): + t3.export_for_interop(t2.SharedEnum) + + t2.export_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t2.export_for_interop(t2.Shared) # should be idempotent + t2.export_for_interop(t2.SharedEnum) + + with pytest.raises(RuntimeError, match="type is not foreign"): + t2.import_for_interop(t2.Shared) + + with pytest.raises(RuntimeError, match=r"is not written in C\+\+"): + t2.import_for_interop(t3.RawShared) + + t2.import_for_interop_explicit(t3.RawShared) + t2.import_for_interop_explicit(t3.RawShared) # should be idempotent + + with pytest.raises( + RuntimeError, match=r"was already imported as a different C\+\+ type" + ): + t2.import_for_interop_wrong_type(t3.RawShared) + + +@pytest.mark.skipif( + (env.MACOS and env.PYPY) or env.ANDROID, + reason="same issue as test_exceptions.py test_cross_module_exception_translator", +) +def test05_interop_exceptions(): + # Once t1 and t2 have registered with pymetabind, which happens as soon as + # they each import or export anything, t1 can translate t2's exceptions. + t1.bind_types() + t2.bind_types() + t1.export_for_interop(t1.Shared) + t2.export_for_interop(t2.Shared) + with pytest.raises(ValueError, match="Shared.123"): + t1.throw_shared(123) + + +def test06_interop_with_cpp(): + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + + # Export t1/t2's Shared to t3, but not the enum yet, and not from t3 + t1.export_for_interop(t1.Shared) + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t2.Shared) + expect(t1, t3, "foreign", enum=False) + expect(t2, t3, "foreign", enum=False) + expect(t1, t2, "isolated") + expect(t3, t1, "isolated", enum=None) + expect(t3, t2, "isolated", enum=None) + + # Now export t2.SharedEnum too. Note that t3 doesn't have its own + # definition of SharedEnum yet, so it will use the imported one and create + # t2.SharedEnums. + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + expect(t1, t3, "foreign", enum=False) + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + t1.export_for_interop(t1.SharedEnum) + t3.import_for_interop(t1.SharedEnum) + expect(t1, t3, "foreign") + expect(t2, t1, "isolated") # t1 hasn't imported anything + expect(t2, t3, "foreign") + expect(t3, t1, "isolated") # t3 sends t2.SharedEnums which t1 can't read + + t1.import_for_interop(t2.SharedEnum) + expect(t2, t1, "isolated", enum=True) + expect(t3, t1, "isolated", enum=True) + + t1.import_for_interop(t2.Shared) + expect(t2, t1, "foreign") + + # No one has imported t3.RawShared, so t3->X doesn't work yet + + # t1/t2 each return their local type since it exists (local is always + # preferred). t3 returns its non-pybind extension type because it has + # an import for that one before any of the imports we wrote. + assert type(t1.make(1)) is t1.Shared + assert type(t2.make(2)) is t2.Shared + assert type(t3.make(3)) is t3.RawShared + + +def test07_interop_with_c(): + t1.bind_types() + t3.create_raw_binding() + t1.export_for_interop(t1.SharedEnum) + t3.import_for_interop(t1.SharedEnum) + t1.import_for_interop_explicit(t3.RawShared) + + # Now that t3.RawShared is imported to t1, we can send in the t3->t1 direction. + expect(t3, t1, "foreign") + + +@pytest.mark.skipif(types_are_immortal, reason="can't GC type object on this platform") +def test08_remove_binding(): + t3.create_raw_binding() + t2.import_for_interop_explicit(t3.RawShared) + + # Remove the binding for t3.RawShared. We expect the t2 domain will + # notice the removal and automatically forget about the defunct binding. + delattr_and_ensure_destroyed((t3, "RawShared")) + t3.create_raw_binding() + + t2.bind_types() + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Similarly test removal of t2.Shared / t2.SharedEnum. + delattr_and_ensure_destroyed((t2, "Shared"), (t2, "SharedEnum")) + t2.bind_types() + + expect(t2, t3, "isolated") + expect(t3, t2, "isolated", enum=None) + + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Removing the binding capsule should work just as well as removing + # the type object. + del t2.Shared.__pymetabind_binding__ + del t2.SharedEnum.__pymetabind_binding__ + pytest.gc_collect() + + expect(t2, t3, "isolated") + expect(t3, t2, "isolated", enum=None) + + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + t2.export_for_interop(t2.SharedEnum) + t3.import_for_interop(t2.SharedEnum) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Re-import RawShared and now everything works again. + t2.import_for_interop_explicit(t3.RawShared) + expect(t2, t3, "foreign") + expect(t3, t2, "foreign") + + # Removing the binding capsule should work just as well as removing + # the type object. + del t3.RawShared.__pymetabind_binding__ + pytest.gc_collect() + t3.export_raw_binding() + + # t3.RawShared was removed from the beginning of t3's list for Shared + # and re-added on the end; also remove and re-add t2.Shared so that + # t3.make() continues to return a t3.RawShared + del t2.Shared.__pymetabind_binding__ + t2.export_for_interop(t2.Shared) + t3.import_for_interop(t2.Shared) + + expect(t2, t3, "foreign") + expect(t3, t2, "isolated", enum=True) + + # Re-import RawShared and now everything works again. + t2.import_for_interop_explicit(t3.RawShared) + expect(t2, t3, "foreign") + expect(t3, t2, "foreign") + + +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +def test09_access_binding_concurrently(): + any_failed = False + t3.create_raw_binding() + + def repeatedly_attempt_conversions(): + deadline = time.time() + 1 + while time.time() < deadline: + try: + assert t3.check(t3.make(5)) == 5 + except: + nonlocal any_failed + any_failed = True + raise + + threads = [ + threading.Thread(target=repeatedly_attempt_conversions) for i in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + assert not any_failed + + +@pytest.mark.skipif(not free_threaded, reason="not relevant on non-FT") +def test10_remove_binding_concurrently(): + transitions = 0 + limit = 5000 + + t1.bind_types() + t2.bind_types() + t3.create_raw_binding() + + def repeatedly_remove_and_readd(): + nonlocal transitions + try: + while transitions < limit: + del t3.RawShared.__pymetabind_binding__ + t3.export_raw_binding() + transitions += 1 + except: + transitions = limit + raise + + thread = threading.Thread(target=repeatedly_remove_and_readd) + thread.start() + + num_failed = 0 + num_successful = 0 + + def repeatedly_attempt_conversions(): + nonlocal num_failed + nonlocal num_successful + while transitions < limit: + try: + assert t3.check(t3.make(42)) == 42 + except TypeError: + num_failed += 1 + else: + num_successful += 1 + + try: + threads = [ + threading.Thread(target=repeatedly_attempt_conversions) for i in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + finally: + transitions = limit + thread.join() + + # typical numbers from my machine: with limit=5000, + # num_failed and num_successful are each several 10k's + print(num_failed, num_successful) + assert num_successful > 0 + assert num_failed > 0 + + +def test11_import_export_all(): + # Enable automatic import and export in the t1/t2 domains. + # Still doesn't help with t3->t1/t2 since t3.RawShared is not a C++ type. + t1.import_all() + t1.export_all() + t1.bind_types() + + t2.import_all() + t2.bind_types() + t2.export_all() + + t3.create_raw_binding() + t3.import_for_interop(t1.Shared) + t3.import_for_interop(t2.SharedEnum) + + expect(t1, t2, "foreign") + expect(t1, t3, "foreign", enum=False) # t3 didn't import all or import t1's enum + expect(t2, t1, "foreign") + expect(t2, t3, "isolated", enum=True) # t3 didn't import t2.Shared + t3.import_all() + expect(t2, t3, "foreign") # t3 didn't import t2.Shared + expect(t3, t1, "isolated", enum=True) + + +def test12_implicit(): + # Create four different types of pyobject, all of which have C++ type Shared + t3.create_raw_binding() + t1.bind_types() + t2.bind_types() + s1 = t1.make(10) + s2 = t2.make(11) + s3r = t3.make(12) + t3.bind_types() + s3p = t3.make(13) + + t2.export_all() + t3.export_all() + t1.import_all() + t1.import_for_interop_explicit(t3.RawShared) + + assert type(s1) is t1.Shared + assert type(s2) is t2.Shared + assert type(s3r) is t3.RawShared + assert type(s3p) is t3.Shared + + # Test implicit conversions from foreign types + for idx, obj in enumerate((s1, s2, s3r, s3p)): + val = t1.test_implicit(obj) + assert val == 10 + idx + + # We should only be sharing in the tX->t1 direction, not vice versa + assert t1.check(s2) == 11 + with pytest.raises(TypeError): + t2.check(s1) + + # Now add the other direction + t1.export_all() + t2.import_all() + assert t2.check(s1) == 10 + + +if sys.implementation.name == "graalpy": + # gc.collect() on GraalPy doesn't reliably collect all garbage, even if + # it's run multiple times, so we can't remove bindings once we've created + # them. Reduce the test suite to a set that can cope with that. + del test03_interop_unimported + del test06_interop_with_cpp + del test07_interop_with_c + del test11_import_export_all + del test12_implicit diff --git a/tests/test_interop_1.cpp b/tests/test_interop_1.cpp new file mode 100644 index 0000000000..d3f6693e3c --- /dev/null +++ b/tests/test_interop_1.cpp @@ -0,0 +1,35 @@ +/* + tests/test_interop_1.cpp -- cross-framework interoperability tests + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Use an unrealistically large internals version to isolate the test_interop +// modules from each other and from the rest of the pybind11 tests +#define PYBIND11_INTERNALS_VERSION 100 + +#include + +#include "test_interop.h" + +namespace py = pybind11; + +PYBIND11_MODULE(test_interop_1, m, py::mod_gil_not_used()) { + py::handle hm = m; + Shared::bind_funcs(m); + m.def("bind_types", [hm]() { Shared::bind_types(hm); }); + + m.def("throw_shared", [](int v) { throw SharedExc{v}; }); + + struct Convertible { + int value; + }; + py::class_(m, "Convertible") + .def(py::init([](const Shared &arg) { return Convertible{arg.value}; })) + .def_readonly("value", &Convertible::value); + py::implicitly_convertible(); + m.def("test_implicit", [](Convertible conv) { return conv.value; }); +} diff --git a/tests/test_interop_2.cpp b/tests/test_interop_2.cpp new file mode 100644 index 0000000000..1a4987fd82 --- /dev/null +++ b/tests/test_interop_2.cpp @@ -0,0 +1,35 @@ +/* + tests/test_interop_2.cpp -- cross-framework interoperability tests + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Use an unrealistically large internals version to isolate the test_interop +// modules from each other and from the rest of the pybind11 tests +#define PYBIND11_INTERNALS_VERSION 200 + +#include + +#include "test_interop.h" + +namespace py = pybind11; + +PYBIND11_MODULE(test_interop_2, m, py::mod_gil_not_used()) { + py::handle hm = m; + Shared::bind_funcs(m); + m.def("bind_types", [hm]() { Shared::bind_types(hm); }); + py::register_exception_translator([](std::exception_ptr p) { + try { + std::rethrow_exception(p); + } catch (const SharedExc &s) { + // Instead of just calling PyErr_SetString, exercise the + // path where one translator throws an exception to be handled + // by another. + throw py::value_error(std::string(py::str("Shared({})").format(s.value)).c_str()); + } + }); + m.def("throw_shared", [](int v) { throw SharedExc{v}; }); +} diff --git a/tests/test_interop_3.cpp b/tests/test_interop_3.cpp new file mode 100644 index 0000000000..6ab6be0266 --- /dev/null +++ b/tests/test_interop_3.cpp @@ -0,0 +1,256 @@ +/* + tests/test_interop_3.cpp -- cross-framework interoperability tests + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +// Use an unrealistically large internals version to isolate the test_interop +// modules from each other and from the rest of the pybind11 tests +#define PYBIND11_INTERNALS_VERSION 300 + +#include + +#include "test_interop.h" + +#include + +namespace py = pybind11; + +// The following is a manual binding to `struct Shared`, created using the +// CPython C API only. + +struct raw_shared_instance { + PyObject_HEAD + uintptr_t spacer[2]; // ensure instance layout differs from nanobind's + bool destroy; + bool deallocate; + Shared *ptr; + Shared value; + PyObject *weakrefs; +}; + +static void Shared_dealloc(struct raw_shared_instance *self) { + if (self->spacer[0] != 0x5a5a5a5a || self->spacer[1] != 0xa5a5a5a5) { + std::terminate(); // instance corrupted + } + if (self->weakrefs) { + PyObject_ClearWeakRefs((PyObject *) self); + } + if (self->destroy) { + self->ptr->~Shared(); + } + if (self->deallocate) { + free(self->ptr); + } + + PyTypeObject *tp = Py_TYPE((PyObject *) self); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyObject *Shared_new(PyTypeObject *type, Shared *value, pymb_rv_policy rvp) { + struct raw_shared_instance *self = PyObject_New(raw_shared_instance, type); + if (self) { + memset((char *) self + sizeof(PyObject), 0, sizeof(*self) - sizeof(PyObject)); + self->spacer[0] = 0x5a5a5a5a; + self->spacer[1] = 0xa5a5a5a5; + switch (rvp) { + case pymb_rv_policy_take_ownership: + self->ptr = value; + self->deallocate = self->destroy = true; + break; + case pymb_rv_policy_copy: + new (&self->value) Shared(*value); + self->ptr = &self->value; + self->deallocate = false; + self->destroy = true; + break; + case pymb_rv_policy_move: + new (&self->value) Shared(std::move(*value)); + self->ptr = &self->value; + self->deallocate = false; + self->destroy = true; + break; + case pymb_rv_policy_reference: + case pymb_rv_policy_share_ownership: + self->ptr = value; + self->deallocate = self->destroy = false; + break; + default: + std::terminate(); // unhandled rvp + } + } + return (PyObject *) self; +} + +static int Shared_init(struct raw_shared_instance *, PyObject *, PyObject *) { + PyErr_SetString(PyExc_TypeError, "cannot be constructed from Python"); + return -1; +} + +// And a minimal implementation for our "foreign framework" of the pymetabind +// interface, so nanobind can use raw_shared_instances. + +static void *hook_from_python(pymb_binding *binding, + PyObject *pyobj, + uint8_t, + void (*)(void *ctx, PyObject *obj), + void *) noexcept { + if (binding->pytype != Py_TYPE(pyobj)) { + return nullptr; + } + return ((raw_shared_instance *) pyobj)->ptr; +} + +static PyObject *hook_to_python(pymb_binding *binding, + void *cobj, + enum pymb_rv_policy rvp, + pymb_to_python_feedback *feedback) noexcept { + feedback->relocate = 0; + if (rvp == pymb_rv_policy_none) { + return nullptr; + } + feedback->is_new = 1; + return Shared_new(binding->pytype, (Shared *) cobj, rvp); +} + +static int hook_no_keep_alive(PyObject *, void *, void (*)(void *)) noexcept { return 0; } +static void hook_ignore_binding(pymb_binding *) noexcept {} +static void hook_ignore_framework(pymb_framework *) noexcept {} +static void hook_free_local_binding(pymb_binding *binding) noexcept { delete binding; } + +PYBIND11_MODULE(test_interop_3, m, py::mod_gil_not_used()) { + static PyMemberDef Shared_members[] = { + {"__weaklistoffset__", + T_PYSSIZET, + offsetof(struct raw_shared_instance, weakrefs), + READONLY, + nullptr}, + {nullptr, 0, 0, 0, nullptr}, + }; + static PyType_Slot Shared_slots[] = { + {Py_tp_doc, (void *) const_cast("Shared object")}, + {Py_tp_init, (void *) Shared_init}, + {Py_tp_dealloc, (void *) Shared_dealloc}, + {Py_tp_members, (void *) Shared_members}, + {0, nullptr}, + }; + static PyType_Spec Shared_spec = { + /* name */ "test_interop_3.RawShared", + /* basicsize */ sizeof(struct raw_shared_instance), + /* itemsize */ 0, + /* flags */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + /* slots */ Shared_slots, + }; + + static auto *registry = pymb_get_registry(); + if (!registry) { + throw py::error_already_set(); + } + + static auto *fw = new pymb_framework{}; + fw->name = "example framework for pybind11 tests"; + fw->flags = pymb_framework_leak_safe; + fw->abi_lang = pymb_abi_lang_c; + fw->from_python = hook_from_python; + fw->to_python = hook_to_python; + fw->keep_alive = hook_no_keep_alive; + fw->remove_local_binding = hook_ignore_binding; + fw->free_local_binding = hook_free_local_binding; + fw->add_foreign_binding = hook_ignore_binding; + fw->remove_foreign_binding = hook_ignore_binding; + fw->add_foreign_framework = hook_ignore_framework; + fw->remove_foreign_framework = hook_ignore_framework; + + pymb_add_framework(registry, fw); + int res = Py_AtExit(+[]() { + pymb_remove_framework(fw); + delete fw; + }); + if (res != 0) { + throw py::error_already_set(); + } + + py::handle hm = m; + Shared::bind_funcs(m); + m.def("bind_types", [hm]() { Shared::bind_types(hm); }); + + m.def("export_raw_binding", [hm]() { + auto type = hm.attr("RawShared"); + auto *binding = new pymb_binding{}; + binding->framework = fw; + binding->pytype = (PyTypeObject *) type.ptr(); + binding->source_name = "RawShared"; + pymb_add_binding(binding, /* tp_finalize_will_remove */ 0); + py::import_for_interop(type); + }); + + m.def("create_raw_binding", [hm]() { + if (py::hasattr(hm, "RawShared")) { + hm.attr("export_raw_binding")(); + return; + } + auto *type = (PyTypeObject *) PyType_FromSpec(&Shared_spec); + if (!type) { + throw py::error_already_set(); + } +#if PY_VERSION_HEX < 0x03090000 + // __weaklistoffset__ member wasn't parsed until 3.9 + type->tp_weaklistoffset = offsetof(struct raw_shared_instance, weakrefs); +#endif + hm.attr("RawShared") = py::reinterpret_steal((PyObject *) type); + hm.attr("export_raw_binding")(); + }); + + m.def("clear_interop_bindings", [hm]() { + // NB: this is not a general purpose solution; the bindings removed + // here won't be re-added if `import_all` is called + py::list bound; + pymb_lock_registry(registry); + // NOLINTNEXTLINE(modernize-use-auto) + PYMB_LIST_FOREACH(struct pymb_binding *, binding, registry->bindings) { + bound.append(py::reinterpret_borrow((PyObject *) binding->pytype)); + } + pymb_unlock_registry(registry); + for (auto type : bound) { + py::delattr(type, "__pymetabind_binding__"); + } + + bool bindings_removed = false; + for (int i = 0; i < 5; ++i) { + pymb_lock_registry(registry); + bindings_removed = pymb_list_is_empty(®istry->bindings); + pymb_unlock_registry(registry); + if (bindings_removed) { + break; + } + py::module_::import("gc").attr("collect")(); + } + if (!bindings_removed) { + throw std::runtime_error("Could not remove bindings"); + } + + // Force trying to re-import/export everything the next time + // import/export all are called, in case we removed bindings + // in between unit tests but couldn't destroy their types + // (because types are immortal in this environment) + for (auto key : py::detail::get_python_state_dict()) { + if (key.attr("startswith")("__pybind11").cast() + && key.attr("endswith")("interop").cast()) { + py::capsule cap = py::detail::get_python_state_dict()[key]; + py::detail::interop_internals **ii = cap; + (*ii)->import_all = (*ii)->export_all = false; + } + } + + // Restore the ability for our own create_shared() etc to work + // properly, since that's a foreign type relationship too + if (py::hasattr(hm, "RawShared")) { + hm.attr("export_raw_binding")(); + py::import_for_interop(hm.attr("RawShared")); + } + }); +} diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 627ccc591d..89d697c715 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -92,6 +92,8 @@ def test_independent_subinterpreters(): pytest.skip("Does not have subinterpreter support compiled in") code = """ +import sys +sys.path.append('.') import mod_per_interpreter_gil as m import pickle with open(pipeo, 'wb') as f: @@ -100,7 +102,10 @@ def test_independent_subinterpreters(): with create() as interp1, create() as interp2: try: - res0 = run_string(interp1, "import mod_shared_interpreter_gil") + res0 = run_string( + interp1, + "import sys; sys.path.append('.'); import mod_shared_interpreter_gil", + ) if res0 is not None: res0 = str(res0) except Exception as e: @@ -141,6 +146,8 @@ def test_independent_subinterpreters_modern(): from concurrent import interpreters code = """ +import sys +sys.path.append('.') import mod_per_interpreter_gil as m values.put_nowait(m.internals_at()) @@ -153,7 +160,9 @@ def test_independent_subinterpreters_modern(): interpreters.ExecutionFailed, match="does not support loading in subinterpreters", ): - interp1.exec("import mod_shared_interpreter_gil") + interp1.exec( + "import sys; sys.path.append('.'); import mod_shared_interpreter_gil" + ) values = interpreters.create_queue() interp1.prepare_main(values=values) @@ -185,6 +194,8 @@ def test_dependent_subinterpreters(): pytest.skip("Does not have subinterpreter support compiled in") code = """ +import sys +sys.path.append('.') import mod_shared_interpreter_gil as m import pickle with open(pipeo, 'wb') as f: