From d378d647384e40d0da6bd3412c833b57d92ebfc9 Mon Sep 17 00:00:00 2001 From: wokron Date: Mon, 23 Feb 2026 22:25:02 +0800 Subject: [PATCH 01/23] zcrx pool basic code --- include/condy/zcrx.hpp | 200 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 include/condy/zcrx.hpp diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp new file mode 100644 index 00000000..5c817df7 --- /dev/null +++ b/include/condy/zcrx.hpp @@ -0,0 +1,200 @@ +#pragma once + +#include "condy/buffers.hpp" +#include "condy/condy_uring.hpp" +#include "condy/context.hpp" +#include "condy/ring.hpp" +#include +#include + +namespace condy { + +class ZeroCopyRxBufferPool; + +struct ZeroCopyRxBuffer : public BufferBase { +public: + ZeroCopyRxBuffer() = default; + ZeroCopyRxBuffer(void *data, size_t size, ZeroCopyRxBufferPool *pool) + : data_(data), size_(size), pool_(pool) {} + ZeroCopyRxBuffer(ZeroCopyRxBuffer &&other) noexcept + : data_(std::exchange(other.data_, nullptr)), + size_(std::exchange(other.size_, 0)), + pool_(std::exchange(other.pool_, nullptr)) {} + ZeroCopyRxBuffer &operator=(ZeroCopyRxBuffer &&other) noexcept { + if (this != &other) { + data_ = std::exchange(other.data_, nullptr); + size_ = std::exchange(other.size_, 0); + pool_ = std::exchange(other.pool_, nullptr); + } + return *this; + } + + ~ZeroCopyRxBuffer() { reset(); } + + ZeroCopyRxBuffer(const ZeroCopyRxBuffer &) = delete; + ZeroCopyRxBuffer &operator=(const ZeroCopyRxBuffer &) = delete; + +public: + /** + * @brief Get the data pointer of the buffer + */ + void *data() const { return data_; } + + /** * + * @brief Get the size of the buffer + */ + size_t size() const { return size_; } + + /** + * @brief Reset the buffer, returning it to the pool if owned + */ + void reset(); + + /** + * @brief Check if the buffer owns a buffer from a pool. + */ + bool owns_buffer() const { return pool_ != nullptr; } + +private: + void *data_ = nullptr; + size_t size_ = 0; + ZeroCopyRxBufferPool *pool_ = nullptr; +}; + +class ZeroCopyRxBufferPool { +public: + // TODO: support different area types + ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, + size_t area_size) { + area_size_ = area_size; + ring_size_ = get_refill_ring_size_(rq_entries); + + area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); + if (area_ptr_ == MAP_FAILED) { + throw std::bad_alloc(); + } + + io_uring_region_desc region_reg = {}; + region_reg.user_addr = 0; + region_reg.size = ring_size_; + region_reg.flags = 0; + io_uring_zcrx_area_reg area_reg = {}; + area_reg.addr = reinterpret_cast(area_ptr_); + area_reg.len = area_size_; + area_reg.flags = 0; + + io_uring_zcrx_ifq_reg reg = {}; + reg.if_idx = if_idx; + reg.if_rxq = if_rxq; + reg.rq_entries = rq_entries; + reg.area_ptr = reinterpret_cast(&area_reg); + reg.region_ptr = reinterpret_cast(®ion_reg); + + register_ifq_(®); + } + + ~ZeroCopyRxBufferPool() { + int r; + assert(area_ptr_ != nullptr); + r = munmap(area_ptr_, area_size_); + assert(r == 0); + assert(ring_ptr_ != nullptr); + r = munmap(ring_ptr_, ring_size_); + assert(r == 0); + // TODO: Unregister ifq + } + + ZeroCopyRxBufferPool(const ZeroCopyRxBufferPool &) = delete; + ZeroCopyRxBufferPool &operator=(const ZeroCopyRxBufferPool &) = delete; + ZeroCopyRxBufferPool(ZeroCopyRxBufferPool &&) = delete; + ZeroCopyRxBufferPool &operator=(ZeroCopyRxBufferPool &&) = delete; + +public: + uint32_t zcrx_id() const { return zcrx_id_; } + + ZeroCopyRxBuffer handle_finish(io_uring_cqe *cqe) { + if (cqe->res < 0) { + return ZeroCopyRxBuffer(); + } + io_uring_zcrx_cqe *rcqe = + reinterpret_cast(cqe + 1); + void *data = static_cast(area_ptr_) + + (rcqe->off & ~~IORING_ZCRX_AREA_MASK); + size_t size = static_cast(cqe->res); + return ZeroCopyRxBuffer(data, size, this); + } + + void add_buffer_back(void *ptr, size_t size) { + // TODO: check refill queue full + io_uring_zcrx_rqe *rqe; + unsigned rq_mask = rq_ring_.ring_entries - 1; + rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; + rqe->off = (static_cast(ptr) - static_cast(area_ptr_)) | + area_token_; + rqe->len = static_cast(size); + io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); + } + +private: + void register_ifq_(io_uring_zcrx_ifq_reg *reg) { + // NOLINTBEGIN(performance-no-int-to-ptr) + auto *region_reg = + reinterpret_cast(reg->region_ptr); + auto *area_reg = + reinterpret_cast(reg->area_ptr); + // NOLINTEND(performance-no-int-to-ptr) + + auto *ring = detail::Context::current().ring(); + int r = io_uring_register_ifq(ring->ring(), reg); + if (r != 0) { + throw make_system_error("io_uring_register_ifq", -r); + } + + ring_ptr_ = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, + static_cast(region_reg->mmap_offset)); + if (ring_ptr_ == MAP_FAILED) { + throw std::bad_alloc(); + } + rq_ring_.khead = + (unsigned int *)((char *)ring_ptr_ + reg->offsets.head); + rq_ring_.ktail = + (unsigned int *)((char *)ring_ptr_ + reg->offsets.tail); + rq_ring_.rqes = + (struct io_uring_zcrx_rqe *)((char *)ring_ptr_ + reg->offsets.rqes); + rq_ring_.rq_tail = 0; + rq_ring_.ring_entries = reg->rq_entries; + + zcrx_id_ = reg->zcrx_id; + area_token_ = area_reg->rq_area_token; + } + + static size_t get_refill_ring_size_(uint32_t rq_entries) { + constexpr size_t page_size = 4096; // TODO: get system page size + size_t ring_size = rq_entries * sizeof(io_uring_zcrx_rqe); + ring_size += page_size; + ring_size = std::bit_ceil(ring_size); + return ring_size; + } + +private: + void *area_ptr_; + size_t area_size_; + void *ring_ptr_; + size_t ring_size_; + io_uring_zcrx_rq rq_ring_; + uint32_t zcrx_id_; + uint64_t area_token_; +}; + +inline void ZeroCopyRxBuffer::reset() { + if (pool_ != nullptr) { + pool_->add_buffer_back(data_, size_); + } + data_ = nullptr; + size_ = 0; + pool_ = nullptr; +} + +} // namespace condy \ No newline at end of file From 84788418b3aaad1c700f3d9361fd08146509fe1a Mon Sep 17 00:00:00 2001 From: wokron Date: Mon, 23 Feb 2026 22:29:06 +0800 Subject: [PATCH 02/23] add ZeroCopyRxCQEHandler --- include/condy/cqe_handler.hpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/include/condy/cqe_handler.hpp b/include/condy/cqe_handler.hpp index 188feb09..f575bc5b 100644 --- a/include/condy/cqe_handler.hpp +++ b/include/condy/cqe_handler.hpp @@ -10,6 +10,7 @@ #include "condy/concepts.hpp" #include "condy/context.hpp" #include "condy/ring.hpp" +#include "condy/zcrx.hpp" #include #include #include @@ -125,4 +126,24 @@ struct TxTimestampCQEHandler { }; #endif +class ZeroCopyRxCQEHandler { +public: + using ReturnType = std::pair; + + ZeroCopyRxCQEHandler(ZeroCopyRxBufferPool *pool) : pool_(pool) {} + + void handle_cqe(io_uring_cqe *cqe) { + assert(detail::check_cqe32(cqe) && + "Expected big CQE for zero-copy RX operations"); + result_.first = cqe->res; + result_.second = pool_->handle_finish(cqe); + } + + ReturnType extract_result() { return std::move(result_); } + +private: + ReturnType result_; + ZeroCopyRxBufferPool *pool_; +}; + } // namespace condy \ No newline at end of file From 8b51265d4c19aa03ac4cc022d8ae10d7a11768ac Mon Sep 17 00:00:00 2001 From: wokron Date: Mon, 23 Feb 2026 22:31:24 +0800 Subject: [PATCH 03/23] add zcrx.h to header --- include/condy.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/condy.hpp b/include/condy.hpp index e32f1eb2..c1bb4023 100644 --- a/include/condy.hpp +++ b/include/condy.hpp @@ -21,6 +21,7 @@ #include "condy/sync_wait.hpp" // IWYU pragma: export #include "condy/task.hpp" // IWYU pragma: export #include "condy/version.hpp" // IWYU pragma: export +#include "condy/zcrx.hpp" // IWYU pragma: export /** * @brief The main namespace for the Condy library. From f2a4d90cc23df41e50305ab4b4c1bedddcf60e2c Mon Sep 17 00:00:00 2001 From: wokron Date: Mon, 23 Feb 2026 22:46:12 +0800 Subject: [PATCH 04/23] add zcrx async func --- include/condy/async_operations.hpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/include/condy/async_operations.hpp b/include/condy/async_operations.hpp index 4d8b83be..52148cf5 100644 --- a/include/condy/async_operations.hpp +++ b/include/condy/async_operations.hpp @@ -12,6 +12,7 @@ #include "condy/concepts.hpp" #include "condy/condy_uring.hpp" #include "condy/helpers.hpp" +#include "condy/zcrx.hpp" namespace condy { @@ -701,6 +702,33 @@ inline auto async_recv_multishot(Fd sockfd, Buffer &buf, int flags, } #endif +namespace detail { + +inline void prep_recv_zc_multishot(io_uring_sqe *sqe, int fd, size_t len, + uint32_t zcrx_id) { + io_uring_prep_rw(IORING_OP_RECV_ZC, sqe, fd, nullptr, len, 0); + sqe->ioprio |= IORING_RECV_MULTISHOT; + sqe->zcrx_ifq_idx = zcrx_id; +} + +} // namespace detail + +// TODO: Consider the function signature later... +template +inline auto async_recv_zc_multishot(Fd fd, size_t len, + ZeroCopyRxBufferPool &pool, + MultiShotFunc &&func) { + auto zcrx_id = pool.zcrx_id(); + auto prep_func = [=](Ring *ring) { + auto *sqe = ring->get_sqe(); + detail::prep_recv_zc_multishot(sqe, fd, len, zcrx_id); + return sqe; + }; + auto op = build_multishot_op_awaiter( + std::move(prep_func), std::forward(func), &pool); + return detail::maybe_flag_fixed_fd(std::move(op), fd); +} + /** * @brief See io_uring_prep_openat2 */ From 3251060538614e2f730d79421bbc0c4af1e55394 Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Feb 2026 12:52:01 +0800 Subject: [PATCH 05/23] fix build --- include/condy/async_operations.hpp | 4 ++++ include/condy/cqe_handler.hpp | 2 ++ include/condy/zcrx.hpp | 7 +++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/condy/async_operations.hpp b/include/condy/async_operations.hpp index 52148cf5..1abfdb1e 100644 --- a/include/condy/async_operations.hpp +++ b/include/condy/async_operations.hpp @@ -702,6 +702,8 @@ inline auto async_recv_multishot(Fd sockfd, Buffer &buf, int flags, } #endif +#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 + namespace detail { inline void prep_recv_zc_multishot(io_uring_sqe *sqe, int fd, size_t len, @@ -729,6 +731,8 @@ inline auto async_recv_zc_multishot(Fd fd, size_t len, return detail::maybe_flag_fixed_fd(std::move(op), fd); } +#endif + /** * @brief See io_uring_prep_openat2 */ diff --git a/include/condy/cqe_handler.hpp b/include/condy/cqe_handler.hpp index f575bc5b..9c9daf62 100644 --- a/include/condy/cqe_handler.hpp +++ b/include/condy/cqe_handler.hpp @@ -126,6 +126,7 @@ struct TxTimestampCQEHandler { }; #endif +#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 class ZeroCopyRxCQEHandler { public: using ReturnType = std::pair; @@ -145,5 +146,6 @@ class ZeroCopyRxCQEHandler { ReturnType result_; ZeroCopyRxBufferPool *pool_; }; +#endif } // namespace condy \ No newline at end of file diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 5c817df7..376f152f 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -4,11 +4,12 @@ #include "condy/condy_uring.hpp" #include "condy/context.hpp" #include "condy/ring.hpp" -#include #include namespace condy { +#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 + class ZeroCopyRxBufferPool; struct ZeroCopyRxBuffer : public BufferBase { @@ -95,7 +96,7 @@ class ZeroCopyRxBufferPool { } ~ZeroCopyRxBufferPool() { - int r; + [[maybe_unused]] int r; assert(area_ptr_ != nullptr); r = munmap(area_ptr_, area_size_); assert(r == 0); @@ -197,4 +198,6 @@ inline void ZeroCopyRxBuffer::reset() { pool_ = nullptr; } +#endif + } // namespace condy \ No newline at end of file From d55f9fea21f938cf68bfd4708fb0de5a926b5ec2 Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Mar 2026 20:23:38 +0800 Subject: [PATCH 06/23] add rq flush --- include/condy/zcrx.hpp | 45 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 376f152f..dd9c3bc6 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -4,6 +4,7 @@ #include "condy/condy_uring.hpp" #include "condy/context.hpp" #include "condy/ring.hpp" +#include "condy/utils.hpp" #include namespace condy { @@ -67,13 +68,14 @@ class ZeroCopyRxBufferPool { // TODO: support different area types ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, size_t area_size) { - area_size_ = area_size; + // TODO: align to system page size + area_size_ = std::bit_ceil(area_size); ring_size_ = get_refill_ring_size_(rq_entries); area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); if (area_ptr_ == MAP_FAILED) { - throw std::bad_alloc(); + throw make_system_error("mmap"); } io_uring_region_desc region_reg = {}; @@ -112,9 +114,9 @@ class ZeroCopyRxBufferPool { ZeroCopyRxBufferPool &operator=(ZeroCopyRxBufferPool &&) = delete; public: - uint32_t zcrx_id() const { return zcrx_id_; } + uint32_t zcrx_id() const noexcept { return zcrx_id_; } - ZeroCopyRxBuffer handle_finish(io_uring_cqe *cqe) { + ZeroCopyRxBuffer handle_finish(io_uring_cqe *cqe) noexcept { if (cqe->res < 0) { return ZeroCopyRxBuffer(); } @@ -126,8 +128,18 @@ class ZeroCopyRxBufferPool { return ZeroCopyRxBuffer(data, size, this); } - void add_buffer_back(void *ptr, size_t size) { - // TODO: check refill queue full + void add_buffer_back(void *ptr, size_t size) noexcept { + if (rq_nr_queued_() == rq_ring_.ring_entries) { + // Flush the refill queue + auto *ring = detail::Context::current().ring(); + zcrx_ctrl ctrl = {}; + ctrl.zcrx_id = zcrx_id_; + ctrl.op = ZCRX_CTRL_FLUSH_RQ; + [[maybe_unused]] int r = + io_uring_register_zcrx_ctrl_(ring->ring(), &ctrl); + assert(r == 0); + } + assert(rq_nr_queued_() < rq_ring_.ring_entries); io_uring_zcrx_rqe *rqe; unsigned rq_mask = rq_ring_.ring_entries - 1; rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; @@ -156,7 +168,7 @@ class ZeroCopyRxBufferPool { MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, static_cast(region_reg->mmap_offset)); if (ring_ptr_ == MAP_FAILED) { - throw std::bad_alloc(); + throw make_system_error("mmap"); } rq_ring_.khead = (unsigned int *)((char *)ring_ptr_ + reg->offsets.head); @@ -171,7 +183,7 @@ class ZeroCopyRxBufferPool { area_token_ = area_reg->rq_area_token; } - static size_t get_refill_ring_size_(uint32_t rq_entries) { + static size_t get_refill_ring_size_(uint32_t rq_entries) noexcept { constexpr size_t page_size = 4096; // TODO: get system page size size_t ring_size = rq_entries * sizeof(io_uring_zcrx_rqe); ring_size += page_size; @@ -179,6 +191,23 @@ class ZeroCopyRxBufferPool { return ring_size; } + size_t rq_nr_queued_() const noexcept { + return rq_ring_.rq_tail - io_uring_smp_load_acquire(rq_ring_.khead); + } + + static int io_uring_register_zcrx_ctrl_(struct io_uring *ring, + struct zcrx_ctrl *ctrl) noexcept { + unsigned int opcode = IORING_REGISTER_ZCRX_CTRL; + int fd; + if (ring->int_flags & 1) { + opcode |= IORING_REGISTER_USE_REGISTERED_RING; + fd = ring->enter_ring_fd; + } else { + fd = ring->ring_fd; + } + return io_uring_register(fd, opcode, ctrl, 0); + } + private: void *area_ptr_; size_t area_size_; From ddf17331ad61b18dc99cfcd313e8682a5153f474 Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Mar 2026 20:28:19 +0800 Subject: [PATCH 07/23] zcrx align up to page size --- include/condy/utils.hpp | 6 ++++++ include/condy/zcrx.hpp | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/include/condy/utils.hpp b/include/condy/utils.hpp index c6c4b90a..539e08a2 100644 --- a/include/condy/utils.hpp +++ b/include/condy/utils.hpp @@ -227,4 +227,10 @@ std::variant tuple_at(std::tuple &results, size_t idx) { } } +template inline T align_up(T value, size_t alignment) noexcept { + // alignment must be a power of two + assert(alignment > 0 && (alignment & (alignment - 1)) == 0); + return (value + alignment - 1) & ~(alignment - 1); +} + } // namespace condy diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index dd9c3bc6..b575ec91 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -40,22 +40,22 @@ struct ZeroCopyRxBuffer : public BufferBase { /** * @brief Get the data pointer of the buffer */ - void *data() const { return data_; } + void *data() const noexcept { return data_; } /** * * @brief Get the size of the buffer */ - size_t size() const { return size_; } + size_t size() const noexcept { return size_; } /** * @brief Reset the buffer, returning it to the pool if owned */ - void reset(); + void reset() noexcept; /** * @brief Check if the buffer owns a buffer from a pool. */ - bool owns_buffer() const { return pool_ != nullptr; } + bool owns_buffer() const noexcept { return pool_ != nullptr; } private: void *data_ = nullptr; @@ -68,9 +68,9 @@ class ZeroCopyRxBufferPool { // TODO: support different area types ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, size_t area_size) { - // TODO: align to system page size - area_size_ = std::bit_ceil(area_size); - ring_size_ = get_refill_ring_size_(rq_entries); + const size_t page_size = sysconf(_SC_PAGESIZE); + area_size_ = align_up(area_size, page_size); + ring_size_ = get_refill_ring_size_(rq_entries, page_size); area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); @@ -183,11 +183,11 @@ class ZeroCopyRxBufferPool { area_token_ = area_reg->rq_area_token; } - static size_t get_refill_ring_size_(uint32_t rq_entries) noexcept { - constexpr size_t page_size = 4096; // TODO: get system page size + static size_t get_refill_ring_size_(uint32_t rq_entries, + size_t page_size) noexcept { size_t ring_size = rq_entries * sizeof(io_uring_zcrx_rqe); ring_size += page_size; - ring_size = std::bit_ceil(ring_size); + ring_size = align_up(ring_size, page_size); return ring_size; } @@ -218,7 +218,7 @@ class ZeroCopyRxBufferPool { uint64_t area_token_; }; -inline void ZeroCopyRxBuffer::reset() { +inline void ZeroCopyRxBuffer::reset() noexcept { if (pool_ != nullptr) { pool_->add_buffer_back(data_, size_); } From 547211ae56c2d73cc08ee6c892cb35ba21d54457 Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Mar 2026 22:19:23 +0800 Subject: [PATCH 08/23] support dmabuf --- include/condy/zcrx.hpp | 67 ++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index b575ec91..2ddc4e03 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -63,45 +63,57 @@ struct ZeroCopyRxBuffer : public BufferBase { ZeroCopyRxBufferPool *pool_ = nullptr; }; +struct ZeroCopyRxDMABufArea { + int dmabuf_fd; + size_t offset; + size_t area_size; +}; + class ZeroCopyRxBufferPool { public: - // TODO: support different area types + ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, + const ZeroCopyRxDMABufArea &area) { + area_size_ = 0; + area_ptr_ = nullptr; + + io_uring_zcrx_area_reg area_reg = {}; + area_reg.addr = area.offset; + area_reg.len = area.area_size; + area_reg.flags = IORING_ZCRX_AREA_DMABUF; + + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, + sysconf(_SC_PAGESIZE)); + } + ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, size_t area_size) { const size_t page_size = sysconf(_SC_PAGESIZE); - area_size_ = align_up(area_size, page_size); - ring_size_ = get_refill_ring_size_(rq_entries, page_size); + area_size_ = align_up(area_size, page_size); area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); if (area_ptr_ == MAP_FAILED) { throw make_system_error("mmap"); } + auto d = defer([&]() { munmap(area_ptr_, area_size_); }); - io_uring_region_desc region_reg = {}; - region_reg.user_addr = 0; - region_reg.size = ring_size_; - region_reg.flags = 0; io_uring_zcrx_area_reg area_reg = {}; area_reg.addr = reinterpret_cast(area_ptr_); area_reg.len = area_size_; area_reg.flags = 0; - io_uring_zcrx_ifq_reg reg = {}; - reg.if_idx = if_idx; - reg.if_rxq = if_rxq; - reg.rq_entries = rq_entries; - reg.area_ptr = reinterpret_cast(&area_reg); - reg.region_ptr = reinterpret_cast(®ion_reg); + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); - register_ifq_(®); + d.dismiss(); } ~ZeroCopyRxBufferPool() { [[maybe_unused]] int r; - assert(area_ptr_ != nullptr); - r = munmap(area_ptr_, area_size_); - assert(r == 0); + if (area_size_ > 0) { + assert(area_ptr_ != nullptr); + r = munmap(area_ptr_, area_size_); + assert(r == 0); + } assert(ring_ptr_ != nullptr); r = munmap(ring_ptr_, ring_size_); assert(r == 0); @@ -123,7 +135,7 @@ class ZeroCopyRxBufferPool { io_uring_zcrx_cqe *rcqe = reinterpret_cast(cqe + 1); void *data = static_cast(area_ptr_) + - (rcqe->off & ~~IORING_ZCRX_AREA_MASK); + (rcqe->off & ~IORING_ZCRX_AREA_MASK); size_t size = static_cast(cqe->res); return ZeroCopyRxBuffer(data, size, this); } @@ -150,6 +162,24 @@ class ZeroCopyRxBufferPool { } private: + void register_ifq_(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, + io_uring_zcrx_area_reg &area_reg, size_t page_size) { + io_uring_region_desc region_reg = {}; + ring_size_ = get_refill_ring_size_(rq_entries, page_size); + region_reg.user_addr = 0; + region_reg.size = ring_size_; + region_reg.flags = 0; + + io_uring_zcrx_ifq_reg reg = {}; + reg.if_idx = if_idx; + reg.if_rxq = if_rxq; + reg.rq_entries = rq_entries; + reg.area_ptr = reinterpret_cast(&area_reg); + reg.region_ptr = reinterpret_cast(®ion_reg); + + register_ifq_(®); + } + void register_ifq_(io_uring_zcrx_ifq_reg *reg) { // NOLINTBEGIN(performance-no-int-to-ptr) auto *region_reg = @@ -163,6 +193,7 @@ class ZeroCopyRxBufferPool { if (r != 0) { throw make_system_error("io_uring_register_ifq", -r); } + // TODO: unregister ifq if any exception ring_ptr_ = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, From 24461fbba4b0b1fd78369e631337883bec7abf5a Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Mar 2026 22:27:48 +0800 Subject: [PATCH 09/23] accept ZeroCopyRxArea --- include/condy/zcrx.hpp | 46 +++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 2ddc4e03..3d791193 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -63,6 +63,11 @@ struct ZeroCopyRxBuffer : public BufferBase { ZeroCopyRxBufferPool *pool_ = nullptr; }; +struct ZeroCopyRxArea { + void *area_addr = nullptr; + size_t area_size; +}; + struct ZeroCopyRxDMABufArea { int dmabuf_fd; size_t offset; @@ -86,25 +91,38 @@ class ZeroCopyRxBufferPool { } ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, - size_t area_size) { + const ZeroCopyRxArea &area) { const size_t page_size = sysconf(_SC_PAGESIZE); - area_size_ = align_up(area_size, page_size); - area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (area_ptr_ == MAP_FAILED) { - throw make_system_error("mmap"); - } - auto d = defer([&]() { munmap(area_ptr_, area_size_); }); + if (area.area_addr == nullptr) { + area_size_ = align_up(area.area_size, page_size); + area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); + if (area_ptr_ == MAP_FAILED) { + throw make_system_error("mmap"); + } + auto d = defer([&]() { munmap(area_ptr_, area_size_); }); - io_uring_zcrx_area_reg area_reg = {}; - area_reg.addr = reinterpret_cast(area_ptr_); - area_reg.len = area_size_; - area_reg.flags = 0; + io_uring_zcrx_area_reg area_reg = {}; + area_reg.addr = reinterpret_cast(area_ptr_); + area_reg.len = area_size_; + area_reg.flags = 0; + + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); - register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); + d.dismiss(); + } else { + // Not owned, so we don't track the size for unmapping + area_size_ = 0; + area_ptr_ = area.area_addr; - d.dismiss(); + io_uring_zcrx_area_reg area_reg = {}; + area_reg.addr = reinterpret_cast(area_ptr_); + area_reg.len = area.area_size; + area_reg.flags = 0; + + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); + } } ~ZeroCopyRxBufferPool() { From a1186f0932a7b0f373bec2b91ef46def58ba3f2e Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Mar 2026 22:48:34 +0800 Subject: [PATCH 10/23] support device-less mode --- include/condy/zcrx.hpp | 76 ++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 3d791193..14c6499e 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -77,21 +77,39 @@ struct ZeroCopyRxDMABufArea { class ZeroCopyRxBufferPool { public: ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, - const ZeroCopyRxDMABufArea &area) { - area_size_ = 0; - area_ptr_ = nullptr; + const ZeroCopyRxArea &area) + : ZeroCopyRxBufferPool(if_idx, if_rxq, rq_entries, area, 0) {} + ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, + const ZeroCopyRxDMABufArea &area) + : ZeroCopyRxBufferPool(if_idx, if_rxq, rq_entries, area, 0) {} - io_uring_zcrx_area_reg area_reg = {}; - area_reg.addr = area.offset; - area_reg.len = area.area_size; - area_reg.flags = IORING_ZCRX_AREA_DMABUF; + // Device-less constructor, DO NOT use this in production code if you don't know what you are doing. + ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxArea &area) + : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) {} + ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxDMABufArea &area) + : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) {} - register_ifq_(if_idx, if_rxq, rq_entries, area_reg, - sysconf(_SC_PAGESIZE)); + ~ZeroCopyRxBufferPool() { + [[maybe_unused]] int r; + if (area_size_ > 0) { + assert(area_ptr_ != nullptr); + r = munmap(area_ptr_, area_size_); + assert(r == 0); + } + assert(ring_ptr_ != nullptr); + r = munmap(ring_ptr_, ring_size_); + assert(r == 0); + // TODO: Unregister ifq } + ZeroCopyRxBufferPool(const ZeroCopyRxBufferPool &) = delete; + ZeroCopyRxBufferPool &operator=(const ZeroCopyRxBufferPool &) = delete; + ZeroCopyRxBufferPool(ZeroCopyRxBufferPool &&) = delete; + ZeroCopyRxBufferPool &operator=(ZeroCopyRxBufferPool &&) = delete; + +private: ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, - const ZeroCopyRxArea &area) { + const ZeroCopyRxArea &area, uint32_t flags) { const size_t page_size = sysconf(_SC_PAGESIZE); if (area.area_addr == nullptr) { @@ -108,7 +126,8 @@ class ZeroCopyRxBufferPool { area_reg.len = area_size_; area_reg.flags = 0; - register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size, + flags); d.dismiss(); } else { @@ -121,27 +140,24 @@ class ZeroCopyRxBufferPool { area_reg.len = area.area_size; area_reg.flags = 0; - register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size, + flags); } } - ~ZeroCopyRxBufferPool() { - [[maybe_unused]] int r; - if (area_size_ > 0) { - assert(area_ptr_ != nullptr); - r = munmap(area_ptr_, area_size_); - assert(r == 0); - } - assert(ring_ptr_ != nullptr); - r = munmap(ring_ptr_, ring_size_); - assert(r == 0); - // TODO: Unregister ifq - } + ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, + const ZeroCopyRxDMABufArea &area, uint32_t flags) { + area_size_ = 0; + area_ptr_ = nullptr; - ZeroCopyRxBufferPool(const ZeroCopyRxBufferPool &) = delete; - ZeroCopyRxBufferPool &operator=(const ZeroCopyRxBufferPool &) = delete; - ZeroCopyRxBufferPool(ZeroCopyRxBufferPool &&) = delete; - ZeroCopyRxBufferPool &operator=(ZeroCopyRxBufferPool &&) = delete; + io_uring_zcrx_area_reg area_reg = {}; + area_reg.addr = area.offset; + area_reg.len = area.area_size; + area_reg.flags = IORING_ZCRX_AREA_DMABUF; + + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, + sysconf(_SC_PAGESIZE), flags); + } public: uint32_t zcrx_id() const noexcept { return zcrx_id_; } @@ -181,7 +197,8 @@ class ZeroCopyRxBufferPool { private: void register_ifq_(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, - io_uring_zcrx_area_reg &area_reg, size_t page_size) { + io_uring_zcrx_area_reg &area_reg, size_t page_size, + uint32_t flags) { io_uring_region_desc region_reg = {}; ring_size_ = get_refill_ring_size_(rq_entries, page_size); region_reg.user_addr = 0; @@ -194,6 +211,7 @@ class ZeroCopyRxBufferPool { reg.rq_entries = rq_entries; reg.area_ptr = reinterpret_cast(&area_reg); reg.region_ptr = reinterpret_cast(®ion_reg); + reg.flags = flags; register_ifq_(®); } From 0092ef93b40f0bd9eb8ea4954904029a9d3454aa Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 24 Mar 2026 22:52:16 +0800 Subject: [PATCH 11/23] merge register_ifq_ --- include/condy/zcrx.hpp | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 14c6499e..995d255e 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -83,7 +83,8 @@ class ZeroCopyRxBufferPool { const ZeroCopyRxDMABufArea &area) : ZeroCopyRxBufferPool(if_idx, if_rxq, rq_entries, area, 0) {} - // Device-less constructor, DO NOT use this in production code if you don't know what you are doing. + // Device-less constructor, DO NOT use this in production code if you don't + // know what you are doing. ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxArea &area) : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) {} ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxDMABufArea &area) @@ -213,19 +214,8 @@ class ZeroCopyRxBufferPool { reg.region_ptr = reinterpret_cast(®ion_reg); reg.flags = flags; - register_ifq_(®); - } - - void register_ifq_(io_uring_zcrx_ifq_reg *reg) { - // NOLINTBEGIN(performance-no-int-to-ptr) - auto *region_reg = - reinterpret_cast(reg->region_ptr); - auto *area_reg = - reinterpret_cast(reg->area_ptr); - // NOLINTEND(performance-no-int-to-ptr) - auto *ring = detail::Context::current().ring(); - int r = io_uring_register_ifq(ring->ring(), reg); + int r = io_uring_register_ifq(ring->ring(), ®); if (r != 0) { throw make_system_error("io_uring_register_ifq", -r); } @@ -233,21 +223,19 @@ class ZeroCopyRxBufferPool { ring_ptr_ = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, - static_cast(region_reg->mmap_offset)); + static_cast(region_reg.mmap_offset)); if (ring_ptr_ == MAP_FAILED) { throw make_system_error("mmap"); } - rq_ring_.khead = - (unsigned int *)((char *)ring_ptr_ + reg->offsets.head); - rq_ring_.ktail = - (unsigned int *)((char *)ring_ptr_ + reg->offsets.tail); + rq_ring_.khead = (unsigned int *)((char *)ring_ptr_ + reg.offsets.head); + rq_ring_.ktail = (unsigned int *)((char *)ring_ptr_ + reg.offsets.tail); rq_ring_.rqes = - (struct io_uring_zcrx_rqe *)((char *)ring_ptr_ + reg->offsets.rqes); + (struct io_uring_zcrx_rqe *)((char *)ring_ptr_ + reg.offsets.rqes); rq_ring_.rq_tail = 0; - rq_ring_.ring_entries = reg->rq_entries; + rq_ring_.ring_entries = reg.rq_entries; - zcrx_id_ = reg->zcrx_id; - area_token_ = area_reg->rq_area_token; + zcrx_id_ = reg.zcrx_id; + area_token_ = area_reg.rq_area_token; } static size_t get_refill_ring_size_(uint32_t rq_entries, From 6f436955ad74b3c2bc8a76dd8259cb4f7ceb4049 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 25 Mar 2026 12:51:37 +0800 Subject: [PATCH 12/23] device-less + dmabuf not support --- include/condy/zcrx.hpp | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 995d255e..d2f129b0 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -79,16 +79,25 @@ class ZeroCopyRxBufferPool { ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, const ZeroCopyRxArea &area) : ZeroCopyRxBufferPool(if_idx, if_rxq, rq_entries, area, 0) {} - ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, - const ZeroCopyRxDMABufArea &area) - : ZeroCopyRxBufferPool(if_idx, if_rxq, rq_entries, area, 0) {} // Device-less constructor, DO NOT use this in production code if you don't // know what you are doing. ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxArea &area) : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) {} - ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxDMABufArea &area) - : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) {} + + ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, + const ZeroCopyRxDMABufArea &area) { + area_size_ = 0; + area_ptr_ = nullptr; + + io_uring_zcrx_area_reg area_reg = {}; + area_reg.addr = area.offset; + area_reg.len = area.area_size; + area_reg.flags = IORING_ZCRX_AREA_DMABUF; + + register_ifq_(if_idx, if_rxq, rq_entries, area_reg, + sysconf(_SC_PAGESIZE), 0); + } ~ZeroCopyRxBufferPool() { [[maybe_unused]] int r; @@ -146,20 +155,6 @@ class ZeroCopyRxBufferPool { } } - ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, - const ZeroCopyRxDMABufArea &area, uint32_t flags) { - area_size_ = 0; - area_ptr_ = nullptr; - - io_uring_zcrx_area_reg area_reg = {}; - area_reg.addr = area.offset; - area_reg.len = area.area_size; - area_reg.flags = IORING_ZCRX_AREA_DMABUF; - - register_ifq_(if_idx, if_rxq, rq_entries, area_reg, - sysconf(_SC_PAGESIZE), flags); - } - public: uint32_t zcrx_id() const noexcept { return zcrx_id_; } From 7ac5b89c88c5952c9a752ed4c7a7a7e0ccf304a4 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 25 Mar 2026 13:41:00 +0800 Subject: [PATCH 13/23] store ring_ptr inside rq_ring_ --- include/condy/zcrx.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index d2f129b0..065cb7dd 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -106,8 +106,8 @@ class ZeroCopyRxBufferPool { r = munmap(area_ptr_, area_size_); assert(r == 0); } - assert(ring_ptr_ != nullptr); - r = munmap(ring_ptr_, ring_size_); + assert(rq_ring_.ring_ptr != nullptr); + r = munmap(rq_ring_.ring_ptr, ring_size_); assert(r == 0); // TODO: Unregister ifq } @@ -216,18 +216,19 @@ class ZeroCopyRxBufferPool { } // TODO: unregister ifq if any exception - ring_ptr_ = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, + void* ring_ptr = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, static_cast(region_reg.mmap_offset)); - if (ring_ptr_ == MAP_FAILED) { + if (ring_ptr == MAP_FAILED) { throw make_system_error("mmap"); } - rq_ring_.khead = (unsigned int *)((char *)ring_ptr_ + reg.offsets.head); - rq_ring_.ktail = (unsigned int *)((char *)ring_ptr_ + reg.offsets.tail); + rq_ring_.khead = (unsigned int *)((char *)ring_ptr + reg.offsets.head); + rq_ring_.ktail = (unsigned int *)((char *)ring_ptr + reg.offsets.tail); rq_ring_.rqes = - (struct io_uring_zcrx_rqe *)((char *)ring_ptr_ + reg.offsets.rqes); + (struct io_uring_zcrx_rqe *)((char *)ring_ptr + reg.offsets.rqes); rq_ring_.rq_tail = 0; rq_ring_.ring_entries = reg.rq_entries; + rq_ring_.ring_ptr = ring_ptr; zcrx_id_ = reg.zcrx_id; area_token_ = area_reg.rq_area_token; @@ -261,7 +262,6 @@ class ZeroCopyRxBufferPool { private: void *area_ptr_; size_t area_size_; - void *ring_ptr_; size_t ring_size_; io_uring_zcrx_rq rq_ring_; uint32_t zcrx_id_; From d0ecebcb5b127a5b7cea792c47b7670c574a55a5 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 25 Mar 2026 13:47:23 +0800 Subject: [PATCH 14/23] simplify field --- include/condy/zcrx.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 065cb7dd..a579dcdf 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -64,14 +64,14 @@ struct ZeroCopyRxBuffer : public BufferBase { }; struct ZeroCopyRxArea { - void *area_addr = nullptr; - size_t area_size; + void *addr = nullptr; + size_t size; }; struct ZeroCopyRxDMABufArea { int dmabuf_fd; size_t offset; - size_t area_size; + size_t size; }; class ZeroCopyRxBufferPool { @@ -92,7 +92,7 @@ class ZeroCopyRxBufferPool { io_uring_zcrx_area_reg area_reg = {}; area_reg.addr = area.offset; - area_reg.len = area.area_size; + area_reg.len = area.size; area_reg.flags = IORING_ZCRX_AREA_DMABUF; register_ifq_(if_idx, if_rxq, rq_entries, area_reg, @@ -122,8 +122,8 @@ class ZeroCopyRxBufferPool { const ZeroCopyRxArea &area, uint32_t flags) { const size_t page_size = sysconf(_SC_PAGESIZE); - if (area.area_addr == nullptr) { - area_size_ = align_up(area.area_size, page_size); + if (area.addr == nullptr) { + area_size_ = align_up(area.size, page_size); area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); if (area_ptr_ == MAP_FAILED) { @@ -143,11 +143,11 @@ class ZeroCopyRxBufferPool { } else { // Not owned, so we don't track the size for unmapping area_size_ = 0; - area_ptr_ = area.area_addr; + area_ptr_ = area.addr; io_uring_zcrx_area_reg area_reg = {}; area_reg.addr = reinterpret_cast(area_ptr_); - area_reg.len = area.area_size; + area_reg.len = area.size; area_reg.flags = 0; register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size, @@ -216,9 +216,9 @@ class ZeroCopyRxBufferPool { } // TODO: unregister ifq if any exception - void* ring_ptr = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, - static_cast(region_reg.mmap_offset)); + void *ring_ptr = mmap(nullptr, ring_size_, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, ring->ring()->ring_fd, + static_cast(region_reg.mmap_offset)); if (ring_ptr == MAP_FAILED) { throw make_system_error("mmap"); } From 2e60e9b0b835177cf180f421ab34fcfc5adfb601 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 25 Mar 2026 14:02:58 +0800 Subject: [PATCH 15/23] rq_entries roundup --- include/condy/zcrx.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index a579dcdf..36314d41 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -195,6 +195,7 @@ class ZeroCopyRxBufferPool { void register_ifq_(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, io_uring_zcrx_area_reg &area_reg, size_t page_size, uint32_t flags) { + rq_entries = std::bit_ceil(rq_entries); io_uring_region_desc region_reg = {}; ring_size_ = get_refill_ring_size_(rq_entries, page_size); region_reg.user_addr = 0; From 399b19f1d7ef78683496967ac23b6d09c1c842d8 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 25 Mar 2026 17:49:28 +0800 Subject: [PATCH 16/23] add simple test case --- include/condy/async_operations.hpp | 12 +++---- include/condy/cqe_handler.hpp | 4 +-- include/condy/zcrx.hpp | 29 +++++++++++----- tests/test_async_operations.4.cpp | 55 ++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/include/condy/async_operations.hpp b/include/condy/async_operations.hpp index 1abfdb1e..0a579ed8 100644 --- a/include/condy/async_operations.hpp +++ b/include/condy/async_operations.hpp @@ -706,9 +706,9 @@ inline auto async_recv_multishot(Fd sockfd, Buffer &buf, int flags, namespace detail { -inline void prep_recv_zc_multishot(io_uring_sqe *sqe, int fd, size_t len, +inline void prep_recv_zc_multishot(io_uring_sqe *sqe, int fd, uint32_t zcrx_id) { - io_uring_prep_rw(IORING_OP_RECV_ZC, sqe, fd, nullptr, len, 0); + io_uring_prep_rw(IORING_OP_RECV_ZC, sqe, fd, nullptr, 0, 0); sqe->ioprio |= IORING_RECV_MULTISHOT; sqe->zcrx_ifq_idx = zcrx_id; } @@ -717,13 +717,13 @@ inline void prep_recv_zc_multishot(io_uring_sqe *sqe, int fd, size_t len, // TODO: Consider the function signature later... template -inline auto async_recv_zc_multishot(Fd fd, size_t len, - ZeroCopyRxBufferPool &pool, - MultiShotFunc &&func) { +inline auto async_recv_multishot(Fd fd, ZeroCopyRxBufferPool &pool, + [[maybe_unused]] int flags, + MultiShotFunc &&func) { auto zcrx_id = pool.zcrx_id(); auto prep_func = [=](Ring *ring) { auto *sqe = ring->get_sqe(); - detail::prep_recv_zc_multishot(sqe, fd, len, zcrx_id); + detail::prep_recv_zc_multishot(sqe, fd, zcrx_id); return sqe; }; auto op = build_multishot_op_awaiter( diff --git a/include/condy/cqe_handler.hpp b/include/condy/cqe_handler.hpp index 9c9daf62..b84101eb 100644 --- a/include/condy/cqe_handler.hpp +++ b/include/condy/cqe_handler.hpp @@ -133,14 +133,14 @@ class ZeroCopyRxCQEHandler { ZeroCopyRxCQEHandler(ZeroCopyRxBufferPool *pool) : pool_(pool) {} - void handle_cqe(io_uring_cqe *cqe) { + void handle_cqe(io_uring_cqe *cqe) noexcept { assert(detail::check_cqe32(cqe) && "Expected big CQE for zero-copy RX operations"); result_.first = cqe->res; result_.second = pool_->handle_finish(cqe); } - ReturnType extract_result() { return std::move(result_); } + ReturnType extract_result() noexcept { return std::move(result_); } private: ReturnType result_; diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 36314d41..54437d77 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -9,6 +9,7 @@ namespace condy { +// TODO: Maybe we need greater version requirements here #if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 class ZeroCopyRxBufferPool; @@ -24,6 +25,7 @@ struct ZeroCopyRxBuffer : public BufferBase { pool_(std::exchange(other.pool_, nullptr)) {} ZeroCopyRxBuffer &operator=(ZeroCopyRxBuffer &&other) noexcept { if (this != &other) { + reset(); data_ = std::exchange(other.data_, nullptr); size_ = std::exchange(other.size_, 0); pool_ = std::exchange(other.pool_, nullptr); @@ -83,7 +85,10 @@ class ZeroCopyRxBufferPool { // Device-less constructor, DO NOT use this in production code if you don't // know what you are doing. ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxArea &area) - : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) {} + : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) { + // TODO: flags should be an enum class + device_less_ = true; + } ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, const ZeroCopyRxDMABufArea &area) { @@ -173,13 +178,7 @@ class ZeroCopyRxBufferPool { void add_buffer_back(void *ptr, size_t size) noexcept { if (rq_nr_queued_() == rq_ring_.ring_entries) { // Flush the refill queue - auto *ring = detail::Context::current().ring(); - zcrx_ctrl ctrl = {}; - ctrl.zcrx_id = zcrx_id_; - ctrl.op = ZCRX_CTRL_FLUSH_RQ; - [[maybe_unused]] int r = - io_uring_register_zcrx_ctrl_(ring->ring(), &ctrl); - assert(r == 0); + flush_rq_(); } assert(rq_nr_queued_() < rq_ring_.ring_entries); io_uring_zcrx_rqe *rqe; @@ -189,6 +188,9 @@ class ZeroCopyRxBufferPool { area_token_; rqe->len = static_cast(size); io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); + if (device_less_) [[unlikely]] { + flush_rq_(); + } } private: @@ -260,6 +262,16 @@ class ZeroCopyRxBufferPool { return io_uring_register(fd, opcode, ctrl, 0); } + void flush_rq_() noexcept { + auto *ring = detail::Context::current().ring(); + zcrx_ctrl ctrl = {}; + ctrl.zcrx_id = zcrx_id_; + ctrl.op = ZCRX_CTRL_FLUSH_RQ; + [[maybe_unused]] int r = + io_uring_register_zcrx_ctrl_(ring->ring(), &ctrl); + assert(r == 0); + } + private: void *area_ptr_; size_t area_size_; @@ -267,6 +279,7 @@ class ZeroCopyRxBufferPool { io_uring_zcrx_rq rq_ring_; uint32_t zcrx_id_; uint64_t area_token_; + bool device_less_ = false; }; inline void ZeroCopyRxBuffer::reset() noexcept { diff --git a/tests/test_async_operations.4.cpp b/tests/test_async_operations.4.cpp index 80e9091e..35971eb5 100644 --- a/tests/test_async_operations.4.cpp +++ b/tests/test_async_operations.4.cpp @@ -4,6 +4,7 @@ #include "condy/runtime.hpp" #include "condy/runtime_options.hpp" #include "condy/sync_wait.hpp" +#include "condy/zcrx.hpp" #include "helpers.hpp" #include #include @@ -1079,4 +1080,58 @@ TEST_CASE("test async_operations - test pipe - direct") { }; condy::sync_wait(func()); } +#endif + +#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 +TEST_CASE("test async_operations - test recv - zc multishot") { + int sv[2]; + create_tcp_socketpair(sv); + + condy::Runtime runtime( + condy::RuntimeOptions().enable_cqe32().enable_defer_taskrun()); + + auto msg = generate_data(9ul * 4096); + ssize_t r = send(sv[1], msg.data(), msg.size(), 0); + REQUIRE(r == msg.size()); + close(sv[1]); + + auto func = [&]() -> condy::Coro { + size_t count = 0; + std::string actual; + + condy::ZeroCopyRxBufferPool pool( + 256, condy::ZeroCopyRxArea{.size = 8ul * 4096}); + + condy::Channel channel(16); + + auto [n, buf] = + co_await condy::async_recv_multishot(sv[0], pool, 0, [&](auto res) { + auto &[n, buf] = res; + REQUIRE(n == 4096); + actual.append(static_cast(buf.data()), n); + count++; + REQUIRE(channel.try_push(std::move(buf)) == 0); + }); + REQUIRE(n == -ENOMEM); + REQUIRE(count == 8); + + auto [r, tmp] = co_await channel.pop(); + tmp.reset(); // Release the buffer back to the pool + + auto [n2, buf2] = + co_await condy::async_recv_multishot(sv[0], pool, 0, [&](auto res) { + auto &[n, buf] = res; + REQUIRE(n == 4096); + actual.append(static_cast(buf.data()), n); + count++; + }); + REQUIRE(n2 == 0); + REQUIRE(count == 9); + + REQUIRE(actual == msg); + }; + condy::sync_wait(runtime, func()); + + close(sv[0]); +} #endif \ No newline at end of file From 4a1bc64bc126227b1c0bb7257ab70a2007e012a5 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 25 Mar 2026 18:51:18 +0800 Subject: [PATCH 17/23] simplify add_buffer_back logic --- include/condy/zcrx.hpp | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 54437d77..d146b399 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -176,19 +176,13 @@ class ZeroCopyRxBufferPool { } void add_buffer_back(void *ptr, size_t size) noexcept { - if (rq_nr_queued_() == rq_ring_.ring_entries) { - // Flush the refill queue - flush_rq_(); - } - assert(rq_nr_queued_() < rq_ring_.ring_entries); - io_uring_zcrx_rqe *rqe; - unsigned rq_mask = rq_ring_.ring_entries - 1; - rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; - rqe->off = (static_cast(ptr) - static_cast(area_ptr_)) | - area_token_; - rqe->len = static_cast(size); - io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); - if (device_less_) [[unlikely]] { + if (!device_less_) [[likely]] { + if (rq_nr_queued_() == rq_ring_.ring_entries) { + flush_rq_(); + } + rq_enqueue_(ptr, size); + } else { + rq_enqueue_(ptr, size); flush_rq_(); } } @@ -262,6 +256,17 @@ class ZeroCopyRxBufferPool { return io_uring_register(fd, opcode, ctrl, 0); } + void rq_enqueue_(void *ptr, size_t size) noexcept { + assert(rq_nr_queued_() < rq_ring_.ring_entries); + io_uring_zcrx_rqe *rqe; + unsigned rq_mask = rq_ring_.ring_entries - 1; + rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; + rqe->off = (static_cast(ptr) - static_cast(area_ptr_)) | + area_token_; + rqe->len = static_cast(size); + io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); + } + void flush_rq_() noexcept { auto *ring = detail::Context::current().ring(); zcrx_ctrl ctrl = {}; From 6d3e9a3950eebf2bec315829e8b6b997d3d47d1e Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 1 Apr 2026 18:03:32 +0800 Subject: [PATCH 18/23] zcrx also use SelectBufferCQEHandler --- include/condy/async_operations.hpp | 3 ++- include/condy/cqe_handler.hpp | 27 +++------------------------ 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/include/condy/async_operations.hpp b/include/condy/async_operations.hpp index 0a579ed8..01675281 100644 --- a/include/condy/async_operations.hpp +++ b/include/condy/async_operations.hpp @@ -726,7 +726,8 @@ inline auto async_recv_multishot(Fd fd, ZeroCopyRxBufferPool &pool, detail::prep_recv_zc_multishot(sqe, fd, zcrx_id); return sqe; }; - auto op = build_multishot_op_awaiter( + auto op = build_multishot_op_awaiter< + SelectBufferCQEHandler>( std::move(prep_func), std::forward(func), &pool); return detail::maybe_flag_fixed_fd(std::move(op), fd); } diff --git a/include/condy/cqe_handler.hpp b/include/condy/cqe_handler.hpp index b84101eb..803e1948 100644 --- a/include/condy/cqe_handler.hpp +++ b/include/condy/cqe_handler.hpp @@ -10,7 +10,6 @@ #include "condy/concepts.hpp" #include "condy/context.hpp" #include "condy/ring.hpp" -#include "condy/zcrx.hpp" #include #include #include @@ -56,7 +55,9 @@ struct SimpleCQEHandler { * result of the operation (the value of `cqe->res`) and the selected buffer, * whose type is determined by the buffer ring. */ -template class SelectBufferCQEHandler { +template + requires(requires(Br *br, io_uring_cqe *cqe) { br->handle_finish(cqe); }) +class SelectBufferCQEHandler { public: SelectBufferCQEHandler(Br *buffers) : buffers_(buffers) {} @@ -126,26 +127,4 @@ struct TxTimestampCQEHandler { }; #endif -#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 -class ZeroCopyRxCQEHandler { -public: - using ReturnType = std::pair; - - ZeroCopyRxCQEHandler(ZeroCopyRxBufferPool *pool) : pool_(pool) {} - - void handle_cqe(io_uring_cqe *cqe) noexcept { - assert(detail::check_cqe32(cqe) && - "Expected big CQE for zero-copy RX operations"); - result_.first = cqe->res; - result_.second = pool_->handle_finish(cqe); - } - - ReturnType extract_result() noexcept { return std::move(result_); } - -private: - ReturnType result_; - ZeroCopyRxBufferPool *pool_; -}; -#endif - } // namespace condy \ No newline at end of file From 7ff85ef7c0e790ddf5619a7a1c5bf0309e5c5063 Mon Sep 17 00:00:00 2001 From: wokron Date: Mon, 13 Apr 2026 11:13:29 +0800 Subject: [PATCH 19/23] simplify rq flush logic --- include/condy/zcrx.hpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index d146b399..5c401748 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -176,15 +176,8 @@ class ZeroCopyRxBufferPool { } void add_buffer_back(void *ptr, size_t size) noexcept { - if (!device_less_) [[likely]] { - if (rq_nr_queued_() == rq_ring_.ring_entries) { - flush_rq_(); - } - rq_enqueue_(ptr, size); - } else { - rq_enqueue_(ptr, size); - flush_rq_(); - } + rq_enqueue_(ptr, size); + maybe_flush_rq_(); } private: @@ -277,6 +270,12 @@ class ZeroCopyRxBufferPool { assert(r == 0); } + void maybe_flush_rq_() noexcept { + if (rq_nr_queued_() >= rq_ring_.ring_entries || device_less_) { + flush_rq_(); + } + } + private: void *area_ptr_; size_t area_size_; From aeaffa01f907a5d8634515bb1972f275e787e56b Mon Sep 17 00:00:00 2001 From: wokron Date: Sat, 18 Apr 2026 22:11:55 +0800 Subject: [PATCH 20/23] use ManagedBuffer for both provided buffer and zcrx --- include/condy/buffers.hpp | 63 +++++++++++++++++++++++++++++ include/condy/provided_buffers.hpp | 64 ++---------------------------- include/condy/zcrx.hpp | 61 +--------------------------- 3 files changed, 67 insertions(+), 121 deletions(-) diff --git a/include/condy/buffers.hpp b/include/condy/buffers.hpp index 155a32ee..771ce8ea 100644 --- a/include/condy/buffers.hpp +++ b/include/condy/buffers.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include namespace condy { @@ -190,4 +191,66 @@ inline MutableBuffer buffer(std::span sp) noexcept { sp.size() * sizeof(PodType)); } +namespace detail { + +template struct ManagedBuffer : public BufferBase { +public: + ManagedBuffer() = default; + ManagedBuffer(void *data, size_t size, BufferPool *pool) + : data_(data), size_(size), pool_(pool) {} + ManagedBuffer(ManagedBuffer &&other) noexcept + : data_(std::exchange(other.data_, nullptr)), + size_(std::exchange(other.size_, 0)), + pool_(std::exchange(other.pool_, nullptr)) {} + ManagedBuffer &operator=(ManagedBuffer &&other) noexcept { + if (this != &other) { + reset(); + data_ = std::exchange(other.data_, nullptr); + size_ = std::exchange(other.size_, 0); + pool_ = std::exchange(other.pool_, nullptr); + } + return *this; + } + + ~ManagedBuffer() { reset(); } + + ManagedBuffer(const ManagedBuffer &) = delete; + ManagedBuffer &operator=(const ManagedBuffer &) = delete; + +public: + /** + * @brief Get the data pointer of the buffer + */ + void *data() const noexcept { return data_; } + + /** * + * @brief Get the size of the buffer + */ + size_t size() const noexcept { return size_; } + + /** + * @brief Reset the buffer, returning it to the pool if owned + */ + void reset() noexcept { + if (pool_ != nullptr) { + pool_->add_buffer_back(data_, size_); + } + data_ = nullptr; + size_ = 0; + pool_ = nullptr; + } + + /** + * @brief Check if the buffer owns a buffer from a pool. + */ + bool owns_buffer() const noexcept { return pool_ != nullptr; } + +private: + void *data_ = nullptr; + size_t size_ = 0; + BufferPool *pool_ = nullptr; +}; + +} // namespace detail + } // namespace condy \ No newline at end of file diff --git a/include/condy/provided_buffers.hpp b/include/condy/provided_buffers.hpp index fc047e7b..e34ac153 100644 --- a/include/condy/provided_buffers.hpp +++ b/include/condy/provided_buffers.hpp @@ -227,57 +227,7 @@ class BundledProvidedBufferPool; * @note The lifetime of the provided buffer must not exceed the lifetime of the * provided buffer pool it is associated with. */ -struct ProvidedBuffer : public BufferBase { -public: - ProvidedBuffer() = default; - ProvidedBuffer(void *data, size_t size, - detail::BundledProvidedBufferPool *pool) - : data_(data), size_(size), pool_(pool) {} - ProvidedBuffer(ProvidedBuffer &&other) noexcept - : data_(std::exchange(other.data_, nullptr)), - size_(std::exchange(other.size_, 0)), - pool_(std::exchange(other.pool_, nullptr)) {} - ProvidedBuffer &operator=(ProvidedBuffer &&other) noexcept { - if (this != &other) { - reset(); - data_ = std::exchange(other.data_, nullptr); - size_ = std::exchange(other.size_, 0); - pool_ = std::exchange(other.pool_, nullptr); - } - return *this; - } - - ~ProvidedBuffer() { reset(); } - - ProvidedBuffer(const ProvidedBuffer &) = delete; - ProvidedBuffer &operator=(const ProvidedBuffer &) = delete; - -public: - /** - * @brief Get the data pointer of the provided buffer - */ - void *data() const noexcept { return data_; } - - /** * - * @brief Get the size of the provided buffer - */ - size_t size() const noexcept { return size_; } - - /** - * @brief Reset the provided buffer, returning it to the pool if owned - */ - void reset() noexcept; - - /** - * @brief Check if the provided buffer owns a buffer from a pool. - */ - bool owns_buffer() const noexcept { return pool_ != nullptr; } - -private: - void *data_ = nullptr; - size_t size_ = 0; - detail::BundledProvidedBufferPool *pool_ = nullptr; -}; +using ProvidedBuffer = detail::ManagedBuffer; namespace detail { @@ -397,7 +347,8 @@ class BundledProvidedBufferPool { return buffers; } - void add_buffer_back(void *ptr) noexcept { + void add_buffer_back(void *ptr, [[maybe_unused]] size_t size) noexcept { + assert(size <= buffer_size_); char *base = get_buffers_base_(); assert(ptr >= base); size_t offset = static_cast(ptr) - base; @@ -437,15 +388,6 @@ class BundledProvidedBufferPool { } // namespace detail -inline void ProvidedBuffer::reset() noexcept { - if (pool_ != nullptr) { - pool_->add_buffer_back(data_); - } - data_ = nullptr; - size_ = 0; - pool_ = nullptr; -} - /** * @brief Provided buffer pool. * @details A provided buffer pool manages a pool of buffers that can be used in diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 5c401748..73ab9ace 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -5,7 +5,6 @@ #include "condy/context.hpp" #include "condy/ring.hpp" #include "condy/utils.hpp" -#include namespace condy { @@ -14,56 +13,7 @@ namespace condy { class ZeroCopyRxBufferPool; -struct ZeroCopyRxBuffer : public BufferBase { -public: - ZeroCopyRxBuffer() = default; - ZeroCopyRxBuffer(void *data, size_t size, ZeroCopyRxBufferPool *pool) - : data_(data), size_(size), pool_(pool) {} - ZeroCopyRxBuffer(ZeroCopyRxBuffer &&other) noexcept - : data_(std::exchange(other.data_, nullptr)), - size_(std::exchange(other.size_, 0)), - pool_(std::exchange(other.pool_, nullptr)) {} - ZeroCopyRxBuffer &operator=(ZeroCopyRxBuffer &&other) noexcept { - if (this != &other) { - reset(); - data_ = std::exchange(other.data_, nullptr); - size_ = std::exchange(other.size_, 0); - pool_ = std::exchange(other.pool_, nullptr); - } - return *this; - } - - ~ZeroCopyRxBuffer() { reset(); } - - ZeroCopyRxBuffer(const ZeroCopyRxBuffer &) = delete; - ZeroCopyRxBuffer &operator=(const ZeroCopyRxBuffer &) = delete; - -public: - /** - * @brief Get the data pointer of the buffer - */ - void *data() const noexcept { return data_; } - - /** * - * @brief Get the size of the buffer - */ - size_t size() const noexcept { return size_; } - - /** - * @brief Reset the buffer, returning it to the pool if owned - */ - void reset() noexcept; - - /** - * @brief Check if the buffer owns a buffer from a pool. - */ - bool owns_buffer() const noexcept { return pool_ != nullptr; } - -private: - void *data_ = nullptr; - size_t size_ = 0; - ZeroCopyRxBufferPool *pool_ = nullptr; -}; +using ZeroCopyRxBuffer = detail::ManagedBuffer; struct ZeroCopyRxArea { void *addr = nullptr; @@ -286,15 +236,6 @@ class ZeroCopyRxBufferPool { bool device_less_ = false; }; -inline void ZeroCopyRxBuffer::reset() noexcept { - if (pool_ != nullptr) { - pool_->add_buffer_back(data_, size_); - } - data_ = nullptr; - size_ = 0; - pool_ = nullptr; -} - #endif } // namespace condy \ No newline at end of file From baca902a9f6cdcb628d5661a09aae2393bb8768d Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 22 Apr 2026 10:45:27 +0800 Subject: [PATCH 21/23] use ZCRX_REG_NODEV --- include/condy/zcrx.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 73ab9ace..0c7fff90 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -35,8 +35,7 @@ class ZeroCopyRxBufferPool { // Device-less constructor, DO NOT use this in production code if you don't // know what you are doing. ZeroCopyRxBufferPool(uint32_t rq_entries, const ZeroCopyRxArea &area) - : ZeroCopyRxBufferPool(0, 0, rq_entries, area, 2) { - // TODO: flags should be an enum class + : ZeroCopyRxBufferPool(0, 0, rq_entries, area, ZCRX_REG_NODEV) { device_less_ = true; } From ae4b3694aa13b5d62fcaacbb4957fb9fc94993f2 Mon Sep 17 00:00:00 2001 From: wokron Date: Fri, 24 Apr 2026 22:23:17 +0800 Subject: [PATCH 22/23] zcrx should use 2.15 --- include/condy/async_operations.hpp | 2 +- include/condy/zcrx.hpp | 3 +-- tests/test_async_operations.4.cpp | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/condy/async_operations.hpp b/include/condy/async_operations.hpp index 01675281..b801d59b 100644 --- a/include/condy/async_operations.hpp +++ b/include/condy/async_operations.hpp @@ -702,7 +702,7 @@ inline auto async_recv_multishot(Fd sockfd, Buffer &buf, int flags, } #endif -#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 +#if !IO_URING_CHECK_VERSION(2, 15) // >= 2.15 namespace detail { diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 0c7fff90..ffad4d52 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -8,8 +8,7 @@ namespace condy { -// TODO: Maybe we need greater version requirements here -#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 +#if !IO_URING_CHECK_VERSION(2, 15) // >= 2.15 class ZeroCopyRxBufferPool; diff --git a/tests/test_async_operations.4.cpp b/tests/test_async_operations.4.cpp index 35971eb5..236584e7 100644 --- a/tests/test_async_operations.4.cpp +++ b/tests/test_async_operations.4.cpp @@ -1082,7 +1082,7 @@ TEST_CASE("test async_operations - test pipe - direct") { } #endif -#if !IO_URING_CHECK_VERSION(2, 10) // >= 2.10 +#if !IO_URING_CHECK_VERSION(2, 15) // >= 2.15 TEST_CASE("test async_operations - test recv - zc multishot") { int sv[2]; create_tcp_socketpair(sv); From 51e811dac25acbab985ed05c2356935f95ded035 Mon Sep 17 00:00:00 2001 From: wokron Date: Fri, 24 Apr 2026 22:32:15 +0800 Subject: [PATCH 23/23] add docs for zcrx --- include/condy/zcrx.hpp | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index ffad4d52..bfd909c3 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -12,21 +12,55 @@ namespace condy { class ZeroCopyRxBufferPool; +/** + * @brief Buffer from a ZeroCopyRxBufferPool. + * @details This buffer type is used for buffers obtained from a + * ZeroCopyRxBufferPool. It automatically returns the buffer to the pool when it + * is out of scope. + * @note The lifetime of the buffer must not exceed the lifetime of the + * ZeroCopyRxBufferPool it is associated with. + */ using ZeroCopyRxBuffer = detail::ManagedBuffer; +/** + * @brief Area for zero-copy receive buffers. + */ struct ZeroCopyRxArea { void *addr = nullptr; size_t size; }; +/** + * @brief Area for zero-copy receive buffers using DMA-BUF. + */ struct ZeroCopyRxDMABufArea { int dmabuf_fd; size_t offset; size_t size; }; +/** + * @brief Buffer pool for zero-copy receive buffers. + * @details This buffer pool utilizes the io_uring zcrx feature to provide + * zero-copy receive buffers. It can be used to receive data directly into + * user-space buffers without copying, which can improve performance for + * high-throughput network applications. + * @returns std::pair When passed to async + * operations, the return type will be a pair of the operation result and the + * @ref ZeroCopyRxBuffer. + * @note The lifetime of this pool must not exceed the running period of the + * associated Runtime, and the lifetime of any ZeroCopyRxBuffer obtained from + * this pool must not exceed the lifetime of this pool. + */ class ZeroCopyRxBufferPool { public: + /** + * @brief Construct a new Zero Copy Rx Buffer Pool object + * @param if_idx Network interface index to register the buffer pool with. + * @param if_rxq Receive queue index to register the buffer pool with. + * @param rq_entries Number of receive queue entries. + * @param area Area for zero-copy receive buffers. + */ ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, const ZeroCopyRxArea &area) : ZeroCopyRxBufferPool(if_idx, if_rxq, rq_entries, area, 0) {} @@ -38,6 +72,13 @@ class ZeroCopyRxBufferPool { device_less_ = true; } + /** + * @brief Construct a new Zero Copy Rx Buffer Pool object + * @param if_idx Network interface index to register the buffer pool with. + * @param if_rxq Receive queue index to register the buffer pool with. + * @param rq_entries Number of receive queue entries. + * @param area Area for zero-copy receive buffers using DMA-BUF. + */ ZeroCopyRxBufferPool(uint32_t if_idx, uint32_t if_rxq, uint32_t rq_entries, const ZeroCopyRxDMABufArea &area) { area_size_ = 0;